{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import sys, os, _pickle as pickle\n",
    "import tensorflow as tf\n",
    "import numpy as np\n",
    "import nltk\n",
    "from sklearn.metrics import f1_score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "data_dir = '../data'\n",
    "ckpt_dir = '../checkpoint'\n",
    "word_embd_dir = '../checkpoint/word_embd'\n",
    "model_dir = '../checkpoint/modelv4'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "word_embd_dim = 100\n",
    "pos_embd_dim = 25\n",
    "dep_embd_dim = 25\n",
    "word_vocab_size = 400001\n",
    "pos_vocab_size = 10\n",
    "dep_vocab_size = 21\n",
    "relation_classes = 19\n",
    "word_state_size = 100\n",
    "other_state_size = 50\n",
    "batch_size = 10\n",
    "channels = 3\n",
    "lambda_l2 = 0.0001\n",
    "max_len_path = 10"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "with tf.name_scope(\"input\"):\n",
    "    path_length = tf.placeholder(tf.int32, shape=[2, batch_size], name=\"path1_length\")\n",
    "    word_ids = tf.placeholder(tf.int32, shape=[2, batch_size, max_len_path], name=\"word_ids\")\n",
    "    pos_ids = tf.placeholder(tf.int32, [2, batch_size, max_len_path], name=\"pos_ids\")\n",
    "    dep_ids = tf.placeholder(tf.int32, [2, batch_size, max_len_path], name=\"dep_ids\")\n",
    "    y = tf.placeholder(tf.int32, [batch_size], name=\"y\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "with tf.name_scope(\"word_embedding\"):\n",
    "    W = tf.Variable(tf.constant(0.0, shape=[word_vocab_size, word_embd_dim]), name=\"W\")\n",
    "    embedding_placeholder = tf.placeholder(tf.float32,[word_vocab_size, word_embd_dim])\n",
    "    embedding_init = W.assign(embedding_placeholder)\n",
    "    embedded_word = tf.nn.embedding_lookup(W, word_ids)\n",
    "    word_embedding_saver = tf.train.Saver({\"word_embedding/W\": W})\n",
    "\n",
    "with tf.name_scope(\"pos_embedding\"):\n",
    "    W = tf.Variable(tf.random_uniform([pos_vocab_size, pos_embd_dim]), name=\"W\")\n",
    "    embedded_pos = tf.nn.embedding_lookup(W, pos_ids)\n",
    "    pos_embedding_saver = tf.train.Saver({\"pos_embedding/W\": W})\n",
    "\n",
    "with tf.name_scope(\"dep_embedding\"):\n",
    "    W = tf.Variable(tf.random_uniform([dep_vocab_size, dep_embd_dim]), name=\"W\")\n",
    "    embedded_dep = tf.nn.embedding_lookup(W, dep_ids)\n",
    "    dep_embedding_saver = tf.train.Saver({\"dep_embedding/W\": W})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "with tf.name_scope(\"word_dropout\"):\n",
    "    embedded_word_drop = tf.nn.dropout(embedded_word, 0.3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "word_hidden_state = tf.zeros([batch_size, word_state_size], name='word_hidden_state')\n",
    "word_cell_state = tf.zeros([batch_size, word_state_size], name='word_cell_state')\n",
    "word_init_state = tf.contrib.rnn.LSTMStateTuple(word_hidden_state, word_cell_state)\n",
    "\n",
    "other_hidden_states = tf.zeros([channels-1, batch_size, other_state_size], name=\"hidden_state\")\n",
    "other_cell_states = tf.zeros([channels-1, batch_size, other_state_size], name=\"cell_state\")\n",
    "\n",
    "other_init_states = [tf.contrib.rnn.LSTMStateTuple(other_hidden_states[i], other_cell_states[i]) for i in range(channels-1)]\n",
    "\n",
    "with tf.variable_scope(\"word_lstm1\"):\n",
    "    cell = tf.contrib.rnn.BasicLSTMCell(word_state_size)\n",
    "    state_series, current_state = tf.nn.dynamic_rnn(cell, embedded_word_drop[0], sequence_length=path_length[0], initial_state=word_init_state)\n",
    "    state_series_word1 = tf.reduce_max(state_series, axis=1)\n",
    "\n",
    "with tf.variable_scope(\"word_lstm2\"):\n",
    "    cell = tf.contrib.rnn.BasicLSTMCell(word_state_size)\n",
    "    state_series, current_state = tf.nn.dynamic_rnn(cell, embedded_word_drop[1], sequence_length=path_length[1], initial_state=word_init_state)\n",
    "    state_series_word2 = tf.reduce_max(state_series, axis=1)\n",
    "\n",
    "with tf.variable_scope(\"pos_lstm1\"):\n",
    "    cell = tf.contrib.rnn.BasicLSTMCell(other_state_size)\n",
    "    state_series, current_state = tf.nn.dynamic_rnn(cell, embedded_pos[0], sequence_length=path_length[0],initial_state=other_init_states[0])\n",
    "    state_series_pos1 = tf.reduce_max(state_series, axis=1)\n",
    "\n",
    "with tf.variable_scope(\"pos_lstm2\"):\n",
    "    cell = tf.contrib.rnn.BasicLSTMCell(other_state_size)\n",
    "    state_series, current_state = tf.nn.dynamic_rnn(cell, embedded_pos[1], sequence_length=path_length[1],initial_state=other_init_states[0])\n",
    "    state_series_pos2 = tf.reduce_max(state_series, axis=1)\n",
    "\n",
    "with tf.variable_scope(\"dep_lstm1\"):\n",
    "    cell = tf.contrib.rnn.BasicLSTMCell(other_state_size)\n",
    "    state_series, current_state = tf.nn.dynamic_rnn(cell, embedded_dep[0], sequence_length=path_length[0], initial_state=other_init_states[1])\n",
    "    state_series_dep1 = tf.reduce_max(state_series, axis=1)\n",
    "\n",
    "with tf.variable_scope(\"dep_lstm2\"):\n",
    "    cell = tf.contrib.rnn.BasicLSTMCell(other_state_size)\n",
    "    state_series, current_state = tf.nn.dynamic_rnn(cell, embedded_dep[1], sequence_length=path_length[1], initial_state=other_init_states[1])\n",
    "    state_series_dep2 = tf.reduce_max(state_series, axis=1)\n",
    "\n",
    "state_series1 = tf.concat([state_series_word1, state_series_pos1, state_series_dep1], 1)\n",
    "state_series2 = tf.concat([state_series_word2, state_series_pos2, state_series_dep2], 1)\n",
    "\n",
    "state_series = tf.concat([state_series1, state_series2], 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "with tf.name_scope(\"hidden_layer\"):\n",
    "    W = tf.Variable(tf.truncated_normal([400, 100], -0.1, 0.1), name=\"W\")\n",
    "    b = tf.Variable(tf.zeros([100]), name=\"b\")\n",
    "    y_hidden_layer = tf.matmul(state_series, W) + b"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "with tf.name_scope(\"softmax_layer\"):\n",
    "    W = tf.Variable(tf.truncated_normal([100, relation_classes], -0.1, 0.1), name=\"W\")\n",
    "    b = tf.Variable(tf.zeros([relation_classes]), name=\"b\")\n",
    "    logits = tf.matmul(y_hidden_layer, W) + b\n",
    "    predictions = tf.argmax(logits, 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "tv_all = tf.trainable_variables()\n",
    "tv_regu = []\n",
    "non_reg = [\"word_embedding/W:0\",\"pos_embedding/W:0\",'dep_embedding/W:0',\"global_step:0\",'hidden_layer/b:0','softmax_layer/b:0']\n",
    "for t in tv_all:\n",
    "    if t.name not in non_reg:\n",
    "        if(t.name.find('biases')==-1):\n",
    "            tv_regu.append(t)\n",
    "\n",
    "with tf.name_scope(\"loss\"):\n",
    "    l2_loss = lambda_l2 * tf.reduce_sum([ tf.nn.l2_loss(v) for v in tv_regu ])\n",
    "    loss = tf.reduce_mean(tf.nn.sparse_softmax_cross_entropy_with_logits(logits=logits, labels=y))\n",
    "    total_loss = loss + l2_loss\n",
    "\n",
    "global_step = tf.Variable(0, name=\"global_step\")\n",
    "\n",
    "optimizer = tf.train.AdamOptimizer(0.001).minimize(total_loss, global_step=global_step)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "f = open(data_dir + '/vocab.pkl', 'rb')\n",
    "vocab = pickle.load(f)\n",
    "f.close()\n",
    "\n",
    "word2id = dict((w, i) for i,w in enumerate(vocab))\n",
    "id2word = dict((i, w) for i,w in enumerate(vocab))\n",
    "\n",
    "unknown_token = \"UNKNOWN_TOKEN\"\n",
    "word2id[unknown_token] = word_vocab_size -1\n",
    "id2word[word_vocab_size-1] = unknown_token\n",
    "\n",
    "pos_tags_vocab = []\n",
    "for line in open(data_dir + '/pos_tags.txt'):\n",
    "        pos_tags_vocab.append(line.strip())\n",
    "\n",
    "dep_vocab = []\n",
    "for line in open(data_dir + '/dependency_types.txt'):\n",
    "    dep_vocab.append(line.strip())\n",
    "\n",
    "relation_vocab = []\n",
    "for line in open(data_dir + '/relation_types.txt'):\n",
    "    relation_vocab.append(line.strip())\n",
    "\n",
    "\n",
    "rel2id = dict((w, i) for i,w in enumerate(relation_vocab))\n",
    "id2rel = dict((i, w) for i,w in enumerate(relation_vocab))\n",
    "\n",
    "pos_tag2id = dict((w, i) for i,w in enumerate(pos_tags_vocab))\n",
    "id2pos_tag = dict((i, w) for i,w in enumerate(pos_tags_vocab))\n",
    "\n",
    "dep2id = dict((w, i) for i,w in enumerate(dep_vocab))\n",
    "id2dep = dict((i, w) for i,w in enumerate(dep_vocab))\n",
    "\n",
    "pos_tag2id['OTH'] = 9\n",
    "id2pos_tag[9] = 'OTH'\n",
    "\n",
    "dep2id['OTH'] = 20\n",
    "id2dep[20] = 'OTH'\n",
    "\n",
    "JJ_pos_tags = ['JJ', 'JJR', 'JJS']\n",
    "NN_pos_tags = ['NN', 'NNS', 'NNP', 'NNPS']\n",
    "RB_pos_tags = ['RB', 'RBR', 'RBS']\n",
    "PRP_pos_tags = ['PRP', 'PRP$']\n",
    "VB_pos_tags = ['VB', 'VBD', 'VBG', 'VBN', 'VBP', 'VBZ']\n",
    "_pos_tags = ['CC', 'CD', 'DT', 'IN']\n",
    "\n",
    "def pos_tag(x):\n",
    "    if x in JJ_pos_tags:\n",
    "        return pos_tag2id['JJ']\n",
    "    if x in NN_pos_tags:\n",
    "        return pos_tag2id['NN']\n",
    "    if x in RB_pos_tags:\n",
    "        return pos_tag2id['RB']\n",
    "    if x in PRP_pos_tags:\n",
    "        return pos_tag2id['PRP']\n",
    "    if x in VB_pos_tags:\n",
    "        return pos_tag2id['VB']\n",
    "    if x in _pos_tags:\n",
    "        return pos_tag2id[x]\n",
    "    else:\n",
    "        return 9"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "sess = tf.Session()\n",
    "sess.run(tf.global_variables_initializer())\n",
    "saver = tf.train.Saver()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# f = open('data/word_embedding', 'rb')\n",
    "# word_embedding = pickle.load(f)\n",
    "# f.close()\n",
    "\n",
    "# sess.run(embedding_init, feed_dict={embedding_placeholder:word_embedding})\n",
    "# word_embedding_saver.save(sess, word_embd_dir + '/word_embd')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# model = tf.train.latest_checkpoint(model_dir)\n",
    "# saver.restore(sess, model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "collapsed": true,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "latest_embd = tf.train.latest_checkpoint(word_embd_dir)\n",
    "word_embedding_saver.restore(sess, latest_embd)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "f = open(data_dir + '/train_paths', 'rb')\n",
    "word_p1, word_p2, dep_p1, dep_p2, pos_p1, pos_p2 = pickle.load(f)\n",
    "f.close()\n",
    "\n",
    "relations = []\n",
    "for line in open(data_dir + '/train_relations.txt'):\n",
    "    relations.append(line.strip().split()[1])\n",
    "\n",
    "length = len(word_p1)\n",
    "num_batches = int(length/batch_size)\n",
    "\n",
    "for i in range(length):\n",
    "    for j, word in enumerate(word_p1[i]):\n",
    "        word = word.lower()\n",
    "        word_p1[i][j] = word if word in word2id else unknown_token \n",
    "    for k, word in enumerate(word_p2[i]):\n",
    "        word = word.lower()\n",
    "        word_p2[i][k] = word if word in word2id else unknown_token \n",
    "    for l, d in enumerate(dep_p1[i]):\n",
    "        dep_p1[i][l] = d if d in dep2id else 'OTH'\n",
    "    for m, d in enumerate(dep_p2[i]):\n",
    "        dep_p2[i][m] = d if d in dep2id else 'OTH'\n",
    "\n",
    "word_p1_ids = np.ones([length, max_len_path],dtype=int)\n",
    "word_p2_ids = np.ones([length, max_len_path],dtype=int)\n",
    "pos_p1_ids = np.ones([length, max_len_path],dtype=int)\n",
    "pos_p2_ids = np.ones([length, max_len_path],dtype=int)\n",
    "dep_p1_ids = np.ones([length, max_len_path],dtype=int)\n",
    "dep_p2_ids = np.ones([length, max_len_path],dtype=int)\n",
    "rel_ids = np.array([rel2id[rel] for rel in relations])\n",
    "path1_len = np.array([len(w) for w in word_p1], dtype=int)\n",
    "path2_len = np.array([len(w) for w in word_p2])\n",
    "\n",
    "for i in range(length):\n",
    "    for j, w in enumerate(word_p1[i]):\n",
    "        word_p1_ids[i][j] = word2id[w]\n",
    "    for j, w in enumerate(word_p2[i]):\n",
    "        word_p2_ids[i][j] = word2id[w]\n",
    "    for j, w in enumerate(pos_p1[i]):\n",
    "        pos_p1_ids[i][j] = pos_tag(w)\n",
    "    for j, w in enumerate(pos_p2[i]):\n",
    "        pos_p2_ids[i][j] = pos_tag(w)\n",
    "    for j, w in enumerate(dep_p1[i]):\n",
    "        dep_p1_ids[i][j] = dep2id[w]\n",
    "    for j, w in enumerate(dep_p2[i]):\n",
    "        dep_p2_ids[i][j] = dep2id[w]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step: 32010 loss: 0.465849\n",
      "Step: 32020 loss: 0.989425\n",
      "Step: 32030 loss: 0.274849\n",
      "Step: 32040 loss: 0.509953\n",
      "Step: 32050 loss: 0.149289\n",
      "Step: 32060 loss: 0.112399\n",
      "Step: 32070 loss: 0.116044\n",
      "Step: 32080 loss: 0.11631\n",
      "Step: 32090 loss: 0.164528\n",
      "Step: 32100 loss: 0.200584\n",
      "Step: 32110 loss: 0.339207\n",
      "Step: 32120 loss: 0.248778\n",
      "Step: 32130 loss: 0.259998\n",
      "Step: 32140 loss: 0.135131\n",
      "Step: 32150 loss: 0.120505\n",
      "Step: 32160 loss: 0.401769\n",
      "Step: 32170 loss: 0.189264\n",
      "Step: 32180 loss: 0.489604\n",
      "Step: 32190 loss: 0.912464\n",
      "Step: 32200 loss: 0.122832\n",
      "Step: 32210 loss: 0.113877\n",
      "Step: 32220 loss: 0.12392\n",
      "Step: 32230 loss: 0.210925\n",
      "Step: 32240 loss: 0.828158\n",
      "Step: 32250 loss: 0.235346\n",
      "Step: 32260 loss: 0.141412\n",
      "Step: 32270 loss: 0.336016\n",
      "Step: 32280 loss: 0.220871\n",
      "Step: 32290 loss: 0.125652\n",
      "Step: 32300 loss: 0.226214\n",
      "Step: 32310 loss: 0.489488\n",
      "Step: 32320 loss: 0.114724\n",
      "Step: 32330 loss: 0.10684\n",
      "Step: 32340 loss: 0.363203\n",
      "Step: 32350 loss: 0.700862\n",
      "Step: 32360 loss: 0.188156\n",
      "Step: 32370 loss: 0.10889\n",
      "Step: 32380 loss: 0.209464\n",
      "Step: 32390 loss: 0.118886\n",
      "Step: 32400 loss: 0.441314\n",
      "Step: 32410 loss: 0.164344\n",
      "Step: 32420 loss: 0.296101\n",
      "Step: 32430 loss: 0.319011\n",
      "Step: 32440 loss: 0.244586\n",
      "Step: 32450 loss: 0.400806\n",
      "Step: 32460 loss: 0.113151\n",
      "Step: 32470 loss: 0.771028\n",
      "Step: 32480 loss: 0.233602\n",
      "Step: 32490 loss: 0.111638\n",
      "Step: 32500 loss: 0.250921\n",
      "Step: 32510 loss: 0.147784\n",
      "Step: 32520 loss: 0.239585\n",
      "Step: 32530 loss: 0.176475\n",
      "Step: 32540 loss: 0.142839\n",
      "Step: 32550 loss: 0.354819\n",
      "Step: 32560 loss: 0.130685\n",
      "Step: 32570 loss: 0.117128\n",
      "Step: 32580 loss: 0.232776\n",
      "Step: 32590 loss: 0.382251\n",
      "Step: 32600 loss: 0.122132\n",
      "Step: 32610 loss: 0.141018\n",
      "Step: 32620 loss: 0.164109\n",
      "Step: 32630 loss: 0.15842\n",
      "Step: 32640 loss: 0.168957\n",
      "Step: 32650 loss: 0.127698\n",
      "Step: 32660 loss: 0.238733\n",
      "Step: 32670 loss: 0.135947\n",
      "Step: 32680 loss: 0.182571\n",
      "Step: 32690 loss: 0.50273\n",
      "Step: 32700 loss: 0.115647\n",
      "Step: 32710 loss: 0.261338\n",
      "Step: 32720 loss: 0.142082\n",
      "Step: 32730 loss: 0.318299\n",
      "Step: 32740 loss: 0.676536\n",
      "Step: 32750 loss: 0.406033\n",
      "Step: 32760 loss: 0.229282\n",
      "Step: 32770 loss: 0.358857\n",
      "Step: 32780 loss: 0.174893\n",
      "Step: 32790 loss: 0.139212\n",
      "Step: 32800 loss: 0.462486\n",
      "Step: 32810 loss: 1.12957\n",
      "Step: 32820 loss: 0.857198\n",
      "Step: 32830 loss: 0.479732\n",
      "Step: 32840 loss: 0.409384\n",
      "Step: 32850 loss: 0.252028\n",
      "Step: 32860 loss: 0.152163\n",
      "Step: 32870 loss: 0.130264\n",
      "Step: 32880 loss: 0.114304\n",
      "Step: 32890 loss: 0.126434\n",
      "Step: 32900 loss: 0.114844\n",
      "Step: 32910 loss: 0.12403\n",
      "Step: 32920 loss: 0.265214\n",
      "Step: 32930 loss: 0.192287\n",
      "Step: 32940 loss: 0.120004\n",
      "Step: 32950 loss: 0.13122\n",
      "Step: 32960 loss: 0.137968\n",
      "Step: 32970 loss: 0.116662\n",
      "Step: 32980 loss: 0.500443\n",
      "Step: 32990 loss: 0.172883\n",
      "Step: 33000 loss: 0.120644\n",
      "Saved Model\n",
      "Step: 33010 loss: 0.108831\n",
      "Step: 33020 loss: 0.114961\n",
      "Step: 33030 loss: 0.126999\n",
      "Step: 33040 loss: 0.360635\n",
      "Step: 33050 loss: 0.234659\n",
      "Step: 33060 loss: 0.239793\n",
      "Step: 33070 loss: 0.128852\n",
      "Step: 33080 loss: 0.394993\n",
      "Step: 33090 loss: 0.302854\n",
      "Step: 33100 loss: 0.122283\n",
      "Step: 33110 loss: 0.122735\n",
      "Step: 33120 loss: 0.320005\n",
      "Step: 33130 loss: 0.253248\n",
      "Step: 33140 loss: 0.126418\n",
      "Step: 33150 loss: 0.449298\n",
      "Step: 33160 loss: 0.325809\n",
      "Step: 33170 loss: 0.109112\n",
      "Step: 33180 loss: 0.149867\n",
      "Step: 33190 loss: 0.319307\n",
      "Step: 33200 loss: 0.250695\n",
      "Step: 33210 loss: 0.160408\n",
      "Step: 33220 loss: 0.390558\n",
      "Step: 33230 loss: 0.314243\n",
      "Step: 33240 loss: 0.1184\n",
      "Step: 33250 loss: 0.149369\n",
      "Step: 33260 loss: 0.119585\n",
      "Step: 33270 loss: 0.260449\n",
      "Step: 33280 loss: 0.143767\n",
      "Step: 33290 loss: 0.187256\n",
      "Step: 33300 loss: 0.255579\n",
      "Step: 33310 loss: 0.175294\n",
      "Step: 33320 loss: 0.12393\n",
      "Step: 33330 loss: 0.247389\n",
      "Step: 33340 loss: 0.137349\n",
      "Step: 33350 loss: 0.115715\n",
      "Step: 33360 loss: 0.145307\n",
      "Step: 33370 loss: 0.147895\n",
      "Step: 33380 loss: 0.125712\n",
      "Step: 33390 loss: 0.13708\n",
      "Step: 33400 loss: 0.190492\n",
      "Step: 33410 loss: 0.617156\n",
      "Step: 33420 loss: 0.119334\n",
      "Step: 33430 loss: 0.329882\n",
      "Step: 33440 loss: 0.118391\n",
      "Step: 33450 loss: 0.134554\n",
      "Step: 33460 loss: 0.18364\n",
      "Step: 33470 loss: 0.114371\n",
      "Step: 33480 loss: 0.984779\n",
      "Step: 33490 loss: 0.341884\n",
      "Step: 33500 loss: 0.112219\n",
      "Step: 33510 loss: 0.131619\n",
      "Step: 33520 loss: 0.154329\n",
      "Step: 33530 loss: 0.42572\n",
      "Step: 33540 loss: 0.179692\n",
      "Step: 33550 loss: 0.250754\n",
      "Step: 33560 loss: 0.157288\n",
      "Step: 33570 loss: 0.599301\n",
      "Step: 33580 loss: 0.516815\n",
      "Step: 33590 loss: 0.268875\n",
      "Step: 33600 loss: 0.359187\n",
      "Step: 33610 loss: 0.384888\n",
      "Step: 33620 loss: 0.605939\n",
      "Step: 33630 loss: 0.502201\n",
      "Step: 33640 loss: 0.122538\n",
      "Step: 33650 loss: 0.325051\n",
      "Step: 33660 loss: 0.267845\n",
      "Step: 33670 loss: 0.139689\n",
      "Step: 33680 loss: 0.148517\n",
      "Step: 33690 loss: 0.110981\n",
      "Step: 33700 loss: 0.145956\n",
      "Step: 33710 loss: 0.303739\n",
      "Step: 33720 loss: 1.04289\n",
      "Step: 33730 loss: 0.850255\n",
      "Step: 33740 loss: 0.122888\n",
      "Step: 33750 loss: 0.115435\n",
      "Step: 33760 loss: 0.179408\n",
      "Step: 33770 loss: 0.119395\n",
      "Step: 33780 loss: 0.410937\n",
      "Step: 33790 loss: 0.258396\n",
      "Step: 33800 loss: 0.116786\n",
      "Step: 33810 loss: 0.108007\n",
      "Step: 33820 loss: 0.111229\n",
      "Step: 33830 loss: 0.209696\n",
      "Step: 33840 loss: 0.145879\n",
      "Step: 33850 loss: 0.185129\n",
      "Step: 33860 loss: 0.142557\n",
      "Step: 33870 loss: 0.29089\n",
      "Step: 33880 loss: 0.295237\n",
      "Step: 33890 loss: 0.230894\n",
      "Step: 33900 loss: 0.113345\n",
      "Step: 33910 loss: 0.126338\n",
      "Step: 33920 loss: 0.274677\n",
      "Step: 33930 loss: 0.518361\n",
      "Step: 33940 loss: 0.198409\n",
      "Step: 33950 loss: 0.375335\n",
      "Step: 33960 loss: 0.199356\n",
      "Step: 33970 loss: 0.109639\n",
      "Step: 33980 loss: 0.126584\n",
      "Step: 33990 loss: 0.668433\n",
      "Step: 34000 loss: 0.181866\n",
      "Saved Model\n",
      "Step: 34010 loss: 0.11158\n",
      "Step: 34020 loss: 0.199896\n",
      "Step: 34030 loss: 0.535507\n",
      "Step: 34040 loss: 0.406827\n",
      "Step: 34050 loss: 0.136543\n",
      "Step: 34060 loss: 0.703377\n",
      "Step: 34070 loss: 0.14051\n",
      "Step: 34080 loss: 0.134413\n",
      "Step: 34090 loss: 0.130903\n",
      "Step: 34100 loss: 0.675084\n",
      "Step: 34110 loss: 0.145508\n",
      "Step: 34120 loss: 0.119121\n",
      "Step: 34130 loss: 0.125168\n",
      "Step: 34140 loss: 0.355068\n",
      "Step: 34150 loss: 0.144339\n",
      "Step: 34160 loss: 0.574754\n",
      "Step: 34170 loss: 0.228721\n",
      "Step: 34180 loss: 0.188513\n",
      "Step: 34190 loss: 0.225022\n",
      "Step: 34200 loss: 0.17826\n",
      "Step: 34210 loss: 0.15817\n",
      "Step: 34220 loss: 0.577741\n",
      "Step: 34230 loss: 0.126599\n",
      "Step: 34240 loss: 0.115913\n",
      "Step: 34250 loss: 0.671573\n",
      "Step: 34260 loss: 0.566928\n",
      "Step: 34270 loss: 0.276037\n",
      "Step: 34280 loss: 0.116005\n",
      "Step: 34290 loss: 0.151351\n",
      "Step: 34300 loss: 0.143428\n",
      "Step: 34310 loss: 0.151475\n",
      "Step: 34320 loss: 0.130313\n",
      "Step: 34330 loss: 0.75956\n",
      "Step: 34340 loss: 0.449769\n",
      "Step: 34350 loss: 0.145422\n",
      "Step: 34360 loss: 0.190813\n",
      "Step: 34370 loss: 0.478526\n",
      "Step: 34380 loss: 1.26608\n",
      "Step: 34390 loss: 0.34488\n",
      "Step: 34400 loss: 0.18146\n",
      "Step: 34410 loss: 0.271853\n",
      "Step: 34420 loss: 0.629114\n",
      "Step: 34430 loss: 0.124082\n",
      "Step: 34440 loss: 0.309204\n",
      "Step: 34450 loss: 0.131633\n",
      "Step: 34460 loss: 0.112742\n",
      "Step: 34470 loss: 0.164911\n",
      "Step: 34480 loss: 0.247643\n",
      "Step: 34490 loss: 0.481618\n",
      "Step: 34500 loss: 0.417964\n",
      "Step: 34510 loss: 0.329625\n",
      "Step: 34520 loss: 0.343803\n",
      "Step: 34530 loss: 0.27404\n",
      "Step: 34540 loss: 0.127843\n",
      "Step: 34550 loss: 0.116237\n",
      "Step: 34560 loss: 0.158578\n",
      "Step: 34570 loss: 0.147566\n",
      "Step: 34580 loss: 0.298937\n",
      "Step: 34590 loss: 0.115631\n",
      "Step: 34600 loss: 0.109797\n",
      "Step: 34610 loss: 0.105851\n",
      "Step: 34620 loss: 0.110612\n",
      "Step: 34630 loss: 0.128084\n",
      "Step: 34640 loss: 0.257591\n",
      "Step: 34650 loss: 0.176704\n",
      "Step: 34660 loss: 0.281715\n",
      "Step: 34670 loss: 0.447564\n",
      "Step: 34680 loss: 0.150919\n",
      "Step: 34690 loss: 0.147766\n",
      "Step: 34700 loss: 0.109586\n",
      "Step: 34710 loss: 0.522866\n",
      "Step: 34720 loss: 0.194055\n",
      "Step: 34730 loss: 0.215198\n",
      "Step: 34740 loss: 0.153576\n",
      "Step: 34750 loss: 0.544486\n",
      "Step: 34760 loss: 0.35699\n",
      "Step: 34770 loss: 0.110929\n",
      "Step: 34780 loss: 0.120472\n",
      "Step: 34790 loss: 0.336154\n",
      "Step: 34800 loss: 0.545953\n",
      "Step: 34810 loss: 0.115475\n",
      "Step: 34820 loss: 0.152596\n",
      "Step: 34830 loss: 0.215931\n",
      "Step: 34840 loss: 0.294719\n",
      "Step: 34850 loss: 0.166629\n",
      "Step: 34860 loss: 0.121038\n",
      "Step: 34870 loss: 0.452749\n",
      "Step: 34880 loss: 0.198024\n",
      "Step: 34890 loss: 0.352886\n",
      "Step: 34900 loss: 0.153747\n",
      "Step: 34910 loss: 0.349722\n",
      "Step: 34920 loss: 0.206131\n",
      "Step: 34930 loss: 0.132378\n",
      "Step: 34940 loss: 0.616332\n",
      "Step: 34950 loss: 0.294638\n",
      "Step: 34960 loss: 0.265178\n",
      "Step: 34970 loss: 1.04288\n",
      "Step: 34980 loss: 0.156536\n",
      "Step: 34990 loss: 0.527347\n",
      "Step: 35000 loss: 0.130727\n",
      "Saved Model\n",
      "Step: 35010 loss: 0.186196\n",
      "Step: 35020 loss: 0.117667\n",
      "Step: 35030 loss: 0.191297\n",
      "Step: 35040 loss: 0.216165\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step: 35050 loss: 0.109522\n",
      "Step: 35060 loss: 0.173212\n",
      "Step: 35070 loss: 0.114111\n",
      "Step: 35080 loss: 0.416148\n",
      "Step: 35090 loss: 0.540476\n",
      "Step: 35100 loss: 0.254722\n",
      "Step: 35110 loss: 0.227785\n",
      "Step: 35120 loss: 0.118375\n",
      "Step: 35130 loss: 0.356484\n",
      "Step: 35140 loss: 0.359755\n",
      "Step: 35150 loss: 0.121271\n",
      "Step: 35160 loss: 0.503607\n",
      "Step: 35170 loss: 0.189824\n",
      "Step: 35180 loss: 0.619622\n",
      "Step: 35190 loss: 0.124397\n",
      "Step: 35200 loss: 0.388573\n",
      "Step: 35210 loss: 0.356752\n",
      "Step: 35220 loss: 0.343743\n",
      "Step: 35230 loss: 0.535166\n",
      "Step: 35240 loss: 0.294901\n",
      "Step: 35250 loss: 0.259709\n",
      "Step: 35260 loss: 0.125196\n",
      "Step: 35270 loss: 0.198215\n",
      "Step: 35280 loss: 0.120735\n",
      "Step: 35290 loss: 0.187133\n",
      "Step: 35300 loss: 0.204983\n",
      "Step: 35310 loss: 0.190654\n",
      "Step: 35320 loss: 0.229639\n",
      "Step: 35330 loss: 0.473693\n",
      "Step: 35340 loss: 0.116598\n",
      "Step: 35350 loss: 0.279773\n",
      "Step: 35360 loss: 0.150126\n",
      "Step: 35370 loss: 0.138339\n",
      "Step: 35380 loss: 0.639304\n",
      "Step: 35390 loss: 0.113349\n",
      "Step: 35400 loss: 0.131463\n",
      "Step: 35410 loss: 0.125274\n",
      "Step: 35420 loss: 0.252035\n",
      "Step: 35430 loss: 0.21323\n",
      "Step: 35440 loss: 0.473202\n",
      "Step: 35450 loss: 0.268337\n",
      "Step: 35460 loss: 0.445871\n",
      "Step: 35470 loss: 0.819647\n",
      "Step: 35480 loss: 0.147935\n",
      "Step: 35490 loss: 0.223147\n",
      "Step: 35500 loss: 0.148824\n",
      "Step: 35510 loss: 0.118484\n",
      "Step: 35520 loss: 0.111922\n",
      "Step: 35530 loss: 0.109414\n",
      "Step: 35540 loss: 0.28875\n",
      "Step: 35550 loss: 0.475373\n",
      "Step: 35560 loss: 0.143043\n",
      "Step: 35570 loss: 0.279294\n",
      "Step: 35580 loss: 0.112138\n",
      "Step: 35590 loss: 0.198917\n",
      "Step: 35600 loss: 0.459864\n",
      "Step: 35610 loss: 0.153214\n",
      "Step: 35620 loss: 0.363542\n",
      "Step: 35630 loss: 0.1189\n",
      "Step: 35640 loss: 0.147886\n",
      "Step: 35650 loss: 0.120111\n",
      "Step: 35660 loss: 0.216049\n",
      "Step: 35670 loss: 0.516454\n",
      "Step: 35680 loss: 0.114603\n",
      "Step: 35690 loss: 0.643396\n",
      "Step: 35700 loss: 0.222139\n",
      "Step: 35710 loss: 0.918055\n",
      "Step: 35720 loss: 0.175547\n",
      "Step: 35730 loss: 0.184003\n",
      "Step: 35740 loss: 0.541492\n",
      "Step: 35750 loss: 0.365158\n",
      "Step: 35760 loss: 0.195942\n",
      "Step: 35770 loss: 0.300656\n",
      "Step: 35780 loss: 0.140948\n",
      "Step: 35790 loss: 0.1546\n",
      "Step: 35800 loss: 0.156517\n",
      "Step: 35810 loss: 0.111975\n",
      "Step: 35820 loss: 0.478272\n",
      "Step: 35830 loss: 0.117021\n",
      "Step: 35840 loss: 0.244474\n",
      "Step: 35850 loss: 0.379118\n",
      "Step: 35860 loss: 0.112375\n",
      "Step: 35870 loss: 0.173791\n",
      "Step: 35880 loss: 0.1943\n",
      "Step: 35890 loss: 0.234491\n",
      "Step: 35900 loss: 0.105799\n",
      "Step: 35910 loss: 0.190824\n",
      "Step: 35920 loss: 0.105745\n",
      "Step: 35930 loss: 0.405296\n",
      "Step: 35940 loss: 0.787582\n",
      "Step: 35950 loss: 0.224526\n",
      "Step: 35960 loss: 0.429299\n",
      "Step: 35970 loss: 0.169351\n",
      "Step: 35980 loss: 0.144658\n",
      "Step: 35990 loss: 0.134943\n",
      "Step: 36000 loss: 0.229179\n",
      "Saved Model\n",
      "Step: 36010 loss: 0.119042\n",
      "Step: 36020 loss: 0.358022\n",
      "Step: 36030 loss: 0.329085\n",
      "Step: 36040 loss: 0.219727\n",
      "Step: 36050 loss: 0.440912\n",
      "Step: 36060 loss: 0.10642\n",
      "Step: 36070 loss: 0.263465\n",
      "Step: 36080 loss: 0.201799\n",
      "Step: 36090 loss: 0.153696\n",
      "Step: 36100 loss: 0.27819\n",
      "Step: 36110 loss: 0.188514\n",
      "Step: 36120 loss: 0.61271\n",
      "Step: 36130 loss: 0.120578\n",
      "Step: 36140 loss: 0.108959\n",
      "Step: 36150 loss: 0.2407\n",
      "Step: 36160 loss: 0.164573\n",
      "Step: 36170 loss: 0.125462\n",
      "Step: 36180 loss: 0.982896\n",
      "Step: 36190 loss: 0.206469\n",
      "Step: 36200 loss: 0.445633\n",
      "Step: 36210 loss: 0.104838\n",
      "Step: 36220 loss: 0.11799\n",
      "Step: 36230 loss: 0.27101\n",
      "Step: 36240 loss: 0.414271\n",
      "Step: 36250 loss: 0.143234\n",
      "Step: 36260 loss: 0.161317\n",
      "Step: 36270 loss: 0.321135\n",
      "Step: 36280 loss: 0.145789\n",
      "Step: 36290 loss: 0.249129\n",
      "Step: 36300 loss: 0.108191\n",
      "Step: 36310 loss: 0.106735\n",
      "Step: 36320 loss: 0.122663\n",
      "Step: 36330 loss: 0.252498\n",
      "Step: 36340 loss: 0.254694\n",
      "Step: 36350 loss: 0.607586\n",
      "Step: 36360 loss: 0.589272\n",
      "Step: 36370 loss: 0.175287\n",
      "Step: 36380 loss: 0.537292\n",
      "Step: 36390 loss: 0.119878\n",
      "Step: 36400 loss: 0.257288\n",
      "Step: 36410 loss: 0.10769\n",
      "Step: 36420 loss: 0.230872\n",
      "Step: 36430 loss: 0.275955\n",
      "Step: 36440 loss: 0.129493\n",
      "Step: 36450 loss: 0.160156\n",
      "Step: 36460 loss: 0.302882\n",
      "Step: 36470 loss: 0.321004\n",
      "Step: 36480 loss: 0.15008\n",
      "Step: 36490 loss: 0.36847\n",
      "Step: 36500 loss: 0.115639\n",
      "Step: 36510 loss: 0.657999\n",
      "Step: 36520 loss: 0.157357\n",
      "Step: 36530 loss: 0.285465\n",
      "Step: 36540 loss: 0.150481\n",
      "Step: 36550 loss: 0.108547\n",
      "Step: 36560 loss: 0.18683\n",
      "Step: 36570 loss: 0.179174\n",
      "Step: 36580 loss: 0.321176\n",
      "Step: 36590 loss: 0.206933\n",
      "Step: 36600 loss: 0.314139\n",
      "Step: 36610 loss: 0.110658\n",
      "Step: 36620 loss: 0.131313\n",
      "Step: 36630 loss: 0.122809\n",
      "Step: 36640 loss: 0.172373\n",
      "Step: 36650 loss: 0.127667\n",
      "Step: 36660 loss: 0.107386\n",
      "Step: 36670 loss: 0.232274\n",
      "Step: 36680 loss: 0.25849\n",
      "Step: 36690 loss: 0.129738\n",
      "Step: 36700 loss: 0.122149\n",
      "Step: 36710 loss: 0.167062\n",
      "Step: 36720 loss: 0.160217\n",
      "Step: 36730 loss: 0.464273\n",
      "Step: 36740 loss: 0.426248\n",
      "Step: 36750 loss: 0.118608\n",
      "Step: 36760 loss: 0.110393\n",
      "Step: 36770 loss: 0.566927\n",
      "Step: 36780 loss: 0.395376\n",
      "Step: 36790 loss: 0.172671\n",
      "Step: 36800 loss: 0.127113\n",
      "Step: 36810 loss: 0.24193\n",
      "Step: 36820 loss: 0.948859\n",
      "Step: 36830 loss: 0.431788\n",
      "Step: 36840 loss: 0.110034\n",
      "Step: 36850 loss: 0.121649\n",
      "Step: 36860 loss: 0.113817\n",
      "Step: 36870 loss: 0.11195\n",
      "Step: 36880 loss: 0.194466\n",
      "Step: 36890 loss: 0.113568\n",
      "Step: 36900 loss: 0.17582\n",
      "Step: 36910 loss: 0.607327\n",
      "Step: 36920 loss: 0.989562\n",
      "Step: 36930 loss: 0.133058\n",
      "Step: 36940 loss: 0.122817\n",
      "Step: 36950 loss: 0.120096\n",
      "Step: 36960 loss: 0.389796\n",
      "Step: 36970 loss: 0.11893\n",
      "Step: 36980 loss: 0.642692\n",
      "Step: 36990 loss: 0.242272\n",
      "Step: 37000 loss: 0.13544\n",
      "Saved Model\n",
      "Step: 37010 loss: 0.106384\n",
      "Step: 37020 loss: 0.11041\n",
      "Step: 37030 loss: 0.12033\n",
      "Step: 37040 loss: 0.348148\n",
      "Step: 37050 loss: 0.11514\n",
      "Step: 37060 loss: 0.141362\n",
      "Step: 37070 loss: 0.11452\n",
      "Step: 37080 loss: 0.136533\n",
      "Step: 37090 loss: 0.124029\n",
      "Step: 37100 loss: 0.106948\n",
      "Step: 37110 loss: 0.295641\n",
      "Step: 37120 loss: 0.373601\n",
      "Step: 37130 loss: 0.147733\n",
      "Step: 37140 loss: 0.355147\n",
      "Step: 37150 loss: 0.13803\n",
      "Step: 37160 loss: 0.180438\n",
      "Step: 37170 loss: 0.108536\n",
      "Step: 37180 loss: 0.611006\n",
      "Step: 37190 loss: 0.385795\n",
      "Step: 37200 loss: 0.272498\n",
      "Step: 37210 loss: 0.127216\n",
      "Step: 37220 loss: 0.136589\n",
      "Step: 37230 loss: 0.124447\n",
      "Step: 37240 loss: 0.113941\n",
      "Step: 37250 loss: 0.105049\n",
      "Step: 37260 loss: 0.165062\n",
      "Step: 37270 loss: 0.54253\n",
      "Step: 37280 loss: 0.127936\n",
      "Step: 37290 loss: 0.187062\n",
      "Step: 37300 loss: 0.191742\n",
      "Step: 37310 loss: 0.536236\n",
      "Step: 37320 loss: 0.241269\n",
      "Step: 37330 loss: 0.147874\n",
      "Step: 37340 loss: 0.211575\n",
      "Step: 37350 loss: 0.114725\n",
      "Step: 37360 loss: 0.162763\n",
      "Step: 37370 loss: 0.105391\n",
      "Step: 37380 loss: 0.249198\n",
      "Step: 37390 loss: 0.376093\n",
      "Step: 37400 loss: 0.242639\n",
      "Step: 37410 loss: 0.119471\n",
      "Step: 37420 loss: 0.157851\n",
      "Step: 37430 loss: 0.113263\n",
      "Step: 37440 loss: 0.213528\n",
      "Step: 37450 loss: 0.304178\n",
      "Step: 37460 loss: 0.164217\n",
      "Step: 37470 loss: 0.278975\n",
      "Step: 37480 loss: 0.166825\n",
      "Step: 37490 loss: 0.133992\n",
      "Step: 37500 loss: 0.146536\n",
      "Step: 37510 loss: 0.518658\n",
      "Step: 37520 loss: 0.14163\n",
      "Step: 37530 loss: 0.562277\n",
      "Step: 37540 loss: 0.244278\n",
      "Step: 37550 loss: 0.213781\n",
      "Step: 37560 loss: 0.249969\n",
      "Step: 37570 loss: 0.222995\n",
      "Step: 37580 loss: 0.275047\n",
      "Step: 37590 loss: 0.235299\n",
      "Step: 37600 loss: 0.567579\n",
      "Step: 37610 loss: 0.54291\n",
      "Step: 37620 loss: 0.478187\n",
      "Step: 37630 loss: 0.397824\n",
      "Step: 37640 loss: 0.702757\n",
      "Step: 37650 loss: 0.197107\n",
      "Step: 37660 loss: 0.125499\n",
      "Step: 37670 loss: 0.188802\n",
      "Step: 37680 loss: 0.125509\n",
      "Step: 37690 loss: 0.338808\n",
      "Step: 37700 loss: 0.380125\n",
      "Step: 37710 loss: 0.594132\n",
      "Step: 37720 loss: 0.51879\n",
      "Step: 37730 loss: 0.107672\n",
      "Step: 37740 loss: 0.115681\n",
      "Step: 37750 loss: 0.127908\n",
      "Step: 37760 loss: 0.109372\n",
      "Step: 37770 loss: 0.170952\n",
      "Step: 37780 loss: 0.14468\n",
      "Step: 37790 loss: 0.458495\n",
      "Step: 37800 loss: 0.162425\n",
      "Step: 37810 loss: 0.10619\n",
      "Step: 37820 loss: 0.129669\n",
      "Step: 37830 loss: 0.224119\n",
      "Step: 37840 loss: 0.249601\n",
      "Step: 37850 loss: 0.119028\n",
      "Step: 37860 loss: 0.47738\n",
      "Step: 37870 loss: 0.30602\n",
      "Step: 37880 loss: 0.204373\n",
      "Step: 37890 loss: 0.108597\n",
      "Step: 37900 loss: 0.112023\n",
      "Step: 37910 loss: 0.382327\n",
      "Step: 37920 loss: 0.179505\n",
      "Step: 37930 loss: 0.114418\n",
      "Step: 37940 loss: 0.13942\n",
      "Step: 37950 loss: 0.272716\n",
      "Step: 37960 loss: 0.155922\n",
      "Step: 37970 loss: 0.117306\n",
      "Step: 37980 loss: 0.112493\n",
      "Step: 37990 loss: 0.174803\n",
      "Step: 38000 loss: 0.294984\n",
      "Saved Model\n",
      "Step: 38010 loss: 0.197053\n",
      "Step: 38020 loss: 0.174863\n",
      "Step: 38030 loss: 0.31724\n",
      "Step: 38040 loss: 0.116294\n",
      "Step: 38050 loss: 0.117936\n",
      "Step: 38060 loss: 0.19906\n",
      "Step: 38070 loss: 0.330114\n",
      "Step: 38080 loss: 0.402536\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step: 38090 loss: 0.175709\n",
      "Step: 38100 loss: 0.287245\n",
      "Step: 38110 loss: 0.193154\n",
      "Step: 38120 loss: 0.130799\n",
      "Step: 38130 loss: 0.155663\n",
      "Step: 38140 loss: 0.139956\n",
      "Step: 38150 loss: 0.509048\n",
      "Step: 38160 loss: 0.19132\n",
      "Step: 38170 loss: 0.158713\n",
      "Step: 38180 loss: 0.435439\n",
      "Step: 38190 loss: 0.284737\n",
      "Step: 38200 loss: 0.136775\n",
      "Step: 38210 loss: 0.110083\n",
      "Step: 38220 loss: 0.132664\n",
      "Step: 38230 loss: 0.132588\n",
      "Step: 38240 loss: 0.109532\n",
      "Step: 38250 loss: 0.206175\n",
      "Step: 38260 loss: 0.109232\n",
      "Step: 38270 loss: 0.219585\n",
      "Step: 38280 loss: 0.223173\n",
      "Step: 38290 loss: 0.231729\n",
      "Step: 38300 loss: 0.108458\n",
      "Step: 38310 loss: 0.210494\n",
      "Step: 38320 loss: 0.234982\n",
      "Step: 38330 loss: 0.565062\n",
      "Step: 38340 loss: 0.950462\n",
      "Step: 38350 loss: 0.161208\n",
      "Step: 38360 loss: 0.292838\n",
      "Step: 38370 loss: 0.324109\n",
      "Step: 38380 loss: 0.332624\n",
      "Step: 38390 loss: 0.2493\n",
      "Step: 38400 loss: 0.465916\n",
      "Step: 38410 loss: 0.19226\n",
      "Step: 38420 loss: 0.189594\n",
      "Step: 38430 loss: 0.205738\n",
      "Step: 38440 loss: 0.578031\n",
      "Step: 38450 loss: 0.230732\n",
      "Step: 38460 loss: 0.106227\n",
      "Step: 38470 loss: 0.142772\n",
      "Step: 38480 loss: 0.105936\n",
      "Step: 38490 loss: 0.127389\n",
      "Step: 38500 loss: 0.279044\n",
      "Step: 38510 loss: 0.183949\n",
      "Step: 38520 loss: 0.293024\n",
      "Step: 38530 loss: 0.185982\n",
      "Step: 38540 loss: 0.114138\n",
      "Step: 38550 loss: 0.151796\n",
      "Step: 38560 loss: 0.147082\n",
      "Step: 38570 loss: 0.170275\n",
      "Step: 38580 loss: 1.0383\n",
      "Step: 38590 loss: 0.117221\n",
      "Step: 38600 loss: 0.106148\n",
      "Step: 38610 loss: 0.103375\n",
      "Step: 38620 loss: 0.116259\n",
      "Step: 38630 loss: 0.125306\n",
      "Step: 38640 loss: 0.159616\n",
      "Step: 38650 loss: 0.109082\n",
      "Step: 38660 loss: 0.116378\n",
      "Step: 38670 loss: 0.121801\n",
      "Step: 38680 loss: 0.270023\n",
      "Step: 38690 loss: 0.122061\n",
      "Step: 38700 loss: 0.107896\n",
      "Step: 38710 loss: 0.103792\n",
      "Step: 38720 loss: 0.176831\n",
      "Step: 38730 loss: 0.138063\n",
      "Step: 38740 loss: 0.163574\n",
      "Step: 38750 loss: 0.297364\n",
      "Step: 38760 loss: 0.111559\n",
      "Step: 38770 loss: 0.214173\n",
      "Step: 38780 loss: 0.321983\n",
      "Step: 38790 loss: 0.33408\n",
      "Step: 38800 loss: 0.870128\n",
      "Step: 38810 loss: 0.134074\n",
      "Step: 38820 loss: 0.218347\n",
      "Step: 38830 loss: 0.192145\n",
      "Step: 38840 loss: 0.103367\n",
      "Step: 38850 loss: 0.11535\n",
      "Step: 38860 loss: 0.131481\n",
      "Step: 38870 loss: 0.115241\n",
      "Step: 38880 loss: 0.308872\n",
      "Step: 38890 loss: 0.129524\n",
      "Step: 38900 loss: 0.323189\n",
      "Step: 38910 loss: 0.124924\n",
      "Step: 38920 loss: 0.425549\n",
      "Step: 38930 loss: 0.124492\n",
      "Step: 38940 loss: 0.149092\n",
      "Step: 38950 loss: 0.13594\n",
      "Step: 38960 loss: 0.27939\n",
      "Step: 38970 loss: 0.167166\n",
      "Step: 38980 loss: 0.271868\n",
      "Step: 38990 loss: 0.533662\n",
      "Step: 39000 loss: 0.235632\n",
      "Saved Model\n",
      "Step: 39010 loss: 0.112736\n",
      "Step: 39020 loss: 0.106412\n",
      "Step: 39030 loss: 0.135255\n",
      "Step: 39040 loss: 0.110624\n",
      "Step: 39050 loss: 0.15728\n",
      "Step: 39060 loss: 0.242887\n",
      "Step: 39070 loss: 0.126791\n",
      "Step: 39080 loss: 0.35025\n",
      "Step: 39090 loss: 0.129605\n",
      "Step: 39100 loss: 0.106829\n",
      "Step: 39110 loss: 0.666846\n",
      "Step: 39120 loss: 0.123726\n",
      "Step: 39130 loss: 0.325571\n",
      "Step: 39140 loss: 0.426756\n",
      "Step: 39150 loss: 0.109113\n",
      "Step: 39160 loss: 0.114067\n",
      "Step: 39170 loss: 0.422194\n",
      "Step: 39180 loss: 0.241521\n",
      "Step: 39190 loss: 0.104137\n",
      "Step: 39200 loss: 0.148658\n",
      "Step: 39210 loss: 0.362969\n",
      "Step: 39220 loss: 1.27519\n",
      "Step: 39230 loss: 0.478848\n",
      "Step: 39240 loss: 0.125038\n",
      "Step: 39250 loss: 0.371785\n",
      "Step: 39260 loss: 0.106594\n",
      "Step: 39270 loss: 0.157474\n",
      "Step: 39280 loss: 0.253396\n",
      "Step: 39290 loss: 0.120353\n",
      "Step: 39300 loss: 0.38205\n",
      "Step: 39310 loss: 0.115467\n",
      "Step: 39320 loss: 0.237331\n",
      "Step: 39330 loss: 0.309209\n",
      "Step: 39340 loss: 0.104097\n",
      "Step: 39350 loss: 0.123377\n",
      "Step: 39360 loss: 0.346038\n",
      "Step: 39370 loss: 0.105219\n",
      "Step: 39380 loss: 0.372041\n",
      "Step: 39390 loss: 0.130155\n",
      "Step: 39400 loss: 0.130367\n",
      "Step: 39410 loss: 0.104989\n",
      "Step: 39420 loss: 0.10676\n",
      "Step: 39430 loss: 0.282315\n",
      "Step: 39440 loss: 0.14324\n",
      "Step: 39450 loss: 0.109761\n",
      "Step: 39460 loss: 0.135274\n",
      "Step: 39470 loss: 0.121722\n",
      "Step: 39480 loss: 0.115188\n",
      "Step: 39490 loss: 0.169259\n",
      "Step: 39500 loss: 0.12707\n",
      "Step: 39510 loss: 0.171332\n",
      "Step: 39520 loss: 0.1449\n",
      "Step: 39530 loss: 0.125977\n",
      "Step: 39540 loss: 0.526145\n",
      "Step: 39550 loss: 0.243814\n",
      "Step: 39560 loss: 0.102864\n",
      "Step: 39570 loss: 0.114569\n",
      "Step: 39580 loss: 0.207052\n",
      "Step: 39590 loss: 0.13666\n",
      "Step: 39600 loss: 0.355698\n",
      "Step: 39610 loss: 0.385244\n",
      "Step: 39620 loss: 0.267874\n",
      "Step: 39630 loss: 0.398006\n",
      "Step: 39640 loss: 0.11096\n",
      "Step: 39650 loss: 0.110459\n",
      "Step: 39660 loss: 0.657678\n",
      "Step: 39670 loss: 0.592432\n",
      "Step: 39680 loss: 0.482119\n",
      "Step: 39690 loss: 0.12715\n",
      "Step: 39700 loss: 0.124559\n",
      "Step: 39710 loss: 0.177516\n",
      "Step: 39720 loss: 0.120716\n",
      "Step: 39730 loss: 0.159541\n",
      "Step: 39740 loss: 0.126991\n",
      "Step: 39750 loss: 0.154536\n",
      "Step: 39760 loss: 0.438974\n",
      "Step: 39770 loss: 0.168019\n",
      "Step: 39780 loss: 0.154293\n",
      "Step: 39790 loss: 0.17798\n",
      "Step: 39800 loss: 0.14944\n",
      "Step: 39810 loss: 0.180916\n",
      "Step: 39820 loss: 0.112791\n",
      "Step: 39830 loss: 0.104746\n",
      "Step: 39840 loss: 0.21589\n",
      "Step: 39850 loss: 0.104456\n",
      "Step: 39860 loss: 0.127914\n",
      "Step: 39870 loss: 0.127372\n",
      "Step: 39880 loss: 0.41407\n",
      "Step: 39890 loss: 0.136722\n",
      "Step: 39900 loss: 0.117959\n",
      "Step: 39910 loss: 0.336625\n",
      "Step: 39920 loss: 0.106263\n",
      "Step: 39930 loss: 0.445911\n",
      "Step: 39940 loss: 0.131704\n",
      "Step: 39950 loss: 0.116727\n",
      "Step: 39960 loss: 0.196782\n",
      "Step: 39970 loss: 0.109335\n",
      "Step: 39980 loss: 0.405315\n",
      "Step: 39990 loss: 0.106169\n",
      "Step: 40000 loss: 0.182926\n",
      "Saved Model\n",
      "Step: 40010 loss: 0.225451\n",
      "Step: 40020 loss: 0.411548\n",
      "Step: 40030 loss: 0.121336\n",
      "Step: 40040 loss: 0.131619\n",
      "Step: 40050 loss: 0.130436\n",
      "Step: 40060 loss: 0.12319\n",
      "Step: 40070 loss: 0.124874\n",
      "Step: 40080 loss: 0.109041\n",
      "Step: 40090 loss: 0.533008\n",
      "Step: 40100 loss: 0.147726\n",
      "Step: 40110 loss: 0.515559\n",
      "Step: 40120 loss: 0.206588\n",
      "Step: 40130 loss: 0.138768\n",
      "Step: 40140 loss: 0.175081\n",
      "Step: 40150 loss: 0.110975\n",
      "Step: 40160 loss: 1.00582\n",
      "Step: 40170 loss: 0.169072\n",
      "Step: 40180 loss: 0.171377\n",
      "Step: 40190 loss: 0.106613\n",
      "Step: 40200 loss: 0.115037\n",
      "Step: 40210 loss: 0.106483\n",
      "Step: 40220 loss: 0.342945\n",
      "Step: 40230 loss: 0.225304\n",
      "Step: 40240 loss: 0.224841\n",
      "Step: 40250 loss: 0.104705\n",
      "Step: 40260 loss: 0.236071\n",
      "Step: 40270 loss: 0.49938\n",
      "Step: 40280 loss: 0.397833\n",
      "Step: 40290 loss: 0.105648\n",
      "Step: 40300 loss: 0.104296\n",
      "Step: 40310 loss: 0.133513\n",
      "Step: 40320 loss: 0.105819\n",
      "Step: 40330 loss: 0.128104\n",
      "Step: 40340 loss: 0.267923\n",
      "Step: 40350 loss: 0.153871\n",
      "Step: 40360 loss: 0.103993\n",
      "Step: 40370 loss: 0.111838\n",
      "Step: 40380 loss: 0.252526\n",
      "Step: 40390 loss: 0.610178\n",
      "Step: 40400 loss: 0.118463\n",
      "Step: 40410 loss: 0.153681\n",
      "Step: 40420 loss: 0.128455\n",
      "Step: 40430 loss: 0.134077\n",
      "Step: 40440 loss: 0.110107\n",
      "Step: 40450 loss: 0.105104\n",
      "Step: 40460 loss: 0.451185\n",
      "Step: 40470 loss: 0.297799\n",
      "Step: 40480 loss: 0.74778\n",
      "Step: 40490 loss: 0.108652\n",
      "Step: 40500 loss: 0.132376\n",
      "Step: 40510 loss: 0.226756\n",
      "Step: 40520 loss: 0.132403\n",
      "Step: 40530 loss: 0.114497\n",
      "Step: 40540 loss: 0.245576\n",
      "Step: 40550 loss: 0.207002\n",
      "Step: 40560 loss: 0.262608\n",
      "Step: 40570 loss: 0.159027\n",
      "Step: 40580 loss: 0.397628\n",
      "Step: 40590 loss: 0.300164\n",
      "Step: 40600 loss: 0.275433\n",
      "Step: 40610 loss: 0.117405\n",
      "Step: 40620 loss: 0.182761\n",
      "Step: 40630 loss: 0.106832\n",
      "Step: 40640 loss: 0.843096\n",
      "Step: 40650 loss: 0.10663\n",
      "Step: 40660 loss: 0.33787\n",
      "Step: 40670 loss: 0.10688\n",
      "Step: 40680 loss: 0.109418\n",
      "Step: 40690 loss: 0.255666\n",
      "Step: 40700 loss: 0.1042\n",
      "Step: 40710 loss: 0.152312\n",
      "Step: 40720 loss: 0.122491\n",
      "Step: 40730 loss: 0.61502\n",
      "Step: 40740 loss: 0.307326\n",
      "Step: 40750 loss: 0.107018\n",
      "Step: 40760 loss: 0.216308\n",
      "Step: 40770 loss: 0.471206\n",
      "Step: 40780 loss: 0.659115\n",
      "Step: 40790 loss: 0.196469\n",
      "Step: 40800 loss: 0.19649\n",
      "Step: 40810 loss: 0.132627\n",
      "Step: 40820 loss: 0.716108\n",
      "Step: 40830 loss: 0.159936\n",
      "Step: 40840 loss: 0.351192\n",
      "Step: 40850 loss: 0.10278\n",
      "Step: 40860 loss: 0.111561\n",
      "Step: 40870 loss: 0.122014\n",
      "Step: 40880 loss: 0.145981\n",
      "Step: 40890 loss: 0.106651\n",
      "Step: 40900 loss: 0.196396\n",
      "Step: 40910 loss: 0.357181\n",
      "Step: 40920 loss: 0.583831\n",
      "Step: 40930 loss: 0.107464\n",
      "Step: 40940 loss: 0.104158\n",
      "Step: 40950 loss: 0.25093\n",
      "Step: 40960 loss: 0.342162\n",
      "Step: 40970 loss: 0.161704\n",
      "Step: 40980 loss: 0.424919\n",
      "Step: 40990 loss: 0.12228\n",
      "Step: 41000 loss: 0.130922\n",
      "Saved Model\n",
      "Step: 41010 loss: 0.109184\n",
      "Step: 41020 loss: 0.125797\n",
      "Step: 41030 loss: 0.658367\n",
      "Step: 41040 loss: 0.114393\n",
      "Step: 41050 loss: 0.701151\n",
      "Step: 41060 loss: 0.107844\n",
      "Step: 41070 loss: 0.250755\n",
      "Step: 41080 loss: 0.41304\n",
      "Step: 41090 loss: 0.106728\n",
      "Step: 41100 loss: 0.137393\n",
      "Step: 41110 loss: 0.350289\n",
      "Step: 41120 loss: 0.126686\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step: 41130 loss: 0.196381\n",
      "Step: 41140 loss: 0.226283\n",
      "Step: 41150 loss: 0.189477\n",
      "Step: 41160 loss: 0.109256\n",
      "Step: 41170 loss: 0.160071\n",
      "Step: 41180 loss: 0.197342\n",
      "Step: 41190 loss: 0.135026\n",
      "Step: 41200 loss: 0.270334\n",
      "Step: 41210 loss: 0.109521\n",
      "Step: 41220 loss: 0.134346\n",
      "Step: 41230 loss: 0.117708\n",
      "Step: 41240 loss: 0.102373\n",
      "Step: 41250 loss: 0.117262\n",
      "Step: 41260 loss: 0.183646\n",
      "Step: 41270 loss: 0.197597\n",
      "Step: 41280 loss: 0.259668\n",
      "Step: 41290 loss: 0.167712\n",
      "Step: 41300 loss: 0.107362\n",
      "Step: 41310 loss: 0.15925\n",
      "Step: 41320 loss: 0.163347\n",
      "Step: 41330 loss: 0.175903\n",
      "Step: 41340 loss: 0.168268\n",
      "Step: 41350 loss: 0.104662\n",
      "Step: 41360 loss: 0.127753\n",
      "Step: 41370 loss: 0.134339\n",
      "Step: 41380 loss: 0.105138\n",
      "Step: 41390 loss: 0.11211\n",
      "Step: 41400 loss: 0.168183\n",
      "Step: 41410 loss: 0.10295\n",
      "Step: 41420 loss: 0.117025\n",
      "Step: 41430 loss: 0.116568\n",
      "Step: 41440 loss: 0.650403\n",
      "Step: 41450 loss: 0.123113\n",
      "Step: 41460 loss: 0.138684\n",
      "Step: 41470 loss: 0.226205\n",
      "Step: 41480 loss: 0.107843\n",
      "Step: 41490 loss: 0.106563\n",
      "Step: 41500 loss: 0.133861\n",
      "Step: 41510 loss: 0.164213\n",
      "Step: 41520 loss: 0.744623\n",
      "Step: 41530 loss: 0.413381\n",
      "Step: 41540 loss: 0.166477\n",
      "Step: 41550 loss: 0.142968\n",
      "Step: 41560 loss: 0.113897\n",
      "Step: 41570 loss: 0.354238\n",
      "Step: 41580 loss: 0.106275\n",
      "Step: 41590 loss: 0.693625\n",
      "Step: 41600 loss: 0.348531\n",
      "Step: 41610 loss: 0.217436\n",
      "Step: 41620 loss: 0.159872\n",
      "Step: 41630 loss: 0.154633\n",
      "Step: 41640 loss: 0.498197\n",
      "Step: 41650 loss: 0.105187\n",
      "Step: 41660 loss: 0.11851\n",
      "Step: 41670 loss: 0.2342\n",
      "Step: 41680 loss: 0.829301\n",
      "Step: 41690 loss: 0.180901\n",
      "Step: 41700 loss: 0.110165\n",
      "Step: 41710 loss: 0.162452\n",
      "Step: 41720 loss: 0.189467\n",
      "Step: 41730 loss: 0.270802\n",
      "Step: 41740 loss: 0.389109\n",
      "Step: 41750 loss: 0.120588\n",
      "Step: 41760 loss: 0.243544\n",
      "Step: 41770 loss: 0.108424\n",
      "Step: 41780 loss: 0.127439\n",
      "Step: 41790 loss: 0.104224\n",
      "Step: 41800 loss: 0.132544\n",
      "Step: 41810 loss: 0.103135\n",
      "Step: 41820 loss: 0.143499\n",
      "Step: 41830 loss: 0.129001\n",
      "Step: 41840 loss: 0.433623\n",
      "Step: 41850 loss: 0.171495\n",
      "Step: 41860 loss: 0.195753\n",
      "Step: 41870 loss: 0.106589\n",
      "Step: 41880 loss: 0.134903\n",
      "Step: 41890 loss: 0.132961\n",
      "Step: 41900 loss: 0.102024\n",
      "Step: 41910 loss: 0.207813\n",
      "Step: 41920 loss: 0.122816\n",
      "Step: 41930 loss: 0.124883\n",
      "Step: 41940 loss: 0.11145\n",
      "Step: 41950 loss: 0.307156\n",
      "Step: 41960 loss: 0.36249\n",
      "Step: 41970 loss: 0.115463\n",
      "Step: 41980 loss: 0.115127\n",
      "Step: 41990 loss: 0.412865\n",
      "Step: 42000 loss: 0.118068\n",
      "Saved Model\n",
      "Step: 42010 loss: 0.12806\n",
      "Step: 42020 loss: 0.451756\n",
      "Step: 42030 loss: 0.140081\n",
      "Step: 42040 loss: 0.103193\n",
      "Step: 42050 loss: 0.147563\n",
      "Step: 42060 loss: 0.196216\n",
      "Step: 42070 loss: 0.259907\n",
      "Step: 42080 loss: 0.131123\n",
      "Step: 42090 loss: 0.11532\n",
      "Step: 42100 loss: 0.203967\n",
      "Step: 42110 loss: 0.248117\n",
      "Step: 42120 loss: 0.382406\n",
      "Step: 42130 loss: 0.221082\n",
      "Step: 42140 loss: 0.14154\n",
      "Step: 42150 loss: 0.216415\n",
      "Step: 42160 loss: 0.229058\n",
      "Step: 42170 loss: 0.195036\n",
      "Step: 42180 loss: 0.124865\n",
      "Step: 42190 loss: 0.29092\n",
      "Step: 42200 loss: 0.164094\n",
      "Step: 42210 loss: 0.306623\n",
      "Step: 42220 loss: 0.264577\n",
      "Step: 42230 loss: 0.39256\n",
      "Step: 42240 loss: 0.167724\n",
      "Step: 42250 loss: 0.247645\n",
      "Step: 42260 loss: 0.109513\n",
      "Step: 42270 loss: 0.11898\n",
      "Step: 42280 loss: 0.176631\n",
      "Step: 42290 loss: 0.104442\n",
      "Step: 42300 loss: 0.101615\n",
      "Step: 42310 loss: 0.220636\n",
      "Step: 42320 loss: 0.119429\n",
      "Step: 42330 loss: 0.731477\n",
      "Step: 42340 loss: 0.854248\n",
      "Step: 42350 loss: 0.122512\n",
      "Step: 42360 loss: 0.128703\n",
      "Step: 42370 loss: 0.709977\n",
      "Step: 42380 loss: 0.105542\n",
      "Step: 42390 loss: 0.123718\n",
      "Step: 42400 loss: 0.276151\n",
      "Step: 42410 loss: 0.314335\n",
      "Step: 42420 loss: 0.505497\n",
      "Step: 42430 loss: 0.241282\n",
      "Step: 42440 loss: 0.385346\n",
      "Step: 42450 loss: 0.246071\n",
      "Step: 42460 loss: 0.104522\n",
      "Step: 42470 loss: 0.202496\n",
      "Step: 42480 loss: 0.107163\n",
      "Step: 42490 loss: 0.15804\n",
      "Step: 42500 loss: 0.156564\n",
      "Step: 42510 loss: 0.430611\n",
      "Step: 42520 loss: 0.338087\n",
      "Step: 42530 loss: 0.207515\n",
      "Step: 42540 loss: 0.140701\n",
      "Step: 42550 loss: 0.330681\n",
      "Step: 42560 loss: 0.148287\n",
      "Step: 42570 loss: 0.10231\n",
      "Step: 42580 loss: 0.534787\n",
      "Step: 42590 loss: 0.116483\n",
      "Step: 42600 loss: 0.108801\n",
      "Step: 42610 loss: 0.104952\n",
      "Step: 42620 loss: 0.104286\n",
      "Step: 42630 loss: 0.242867\n",
      "Step: 42640 loss: 0.517332\n",
      "Step: 42650 loss: 0.147322\n",
      "Step: 42660 loss: 0.19702\n",
      "Step: 42670 loss: 0.111156\n",
      "Step: 42680 loss: 0.333781\n",
      "Step: 42690 loss: 0.15333\n",
      "Step: 42700 loss: 0.105637\n",
      "Step: 42710 loss: 0.103669\n",
      "Step: 42720 loss: 0.201112\n",
      "Step: 42730 loss: 0.196888\n",
      "Step: 42740 loss: 0.10484\n",
      "Step: 42750 loss: 0.346214\n",
      "Step: 42760 loss: 0.161571\n",
      "Step: 42770 loss: 0.244422\n",
      "Step: 42780 loss: 0.121861\n",
      "Step: 42790 loss: 0.212727\n",
      "Step: 42800 loss: 0.342921\n",
      "Step: 42810 loss: 0.130416\n",
      "Step: 42820 loss: 0.208075\n",
      "Step: 42830 loss: 0.113472\n",
      "Step: 42840 loss: 0.164558\n",
      "Step: 42850 loss: 0.102809\n",
      "Step: 42860 loss: 0.324756\n",
      "Step: 42870 loss: 0.201871\n",
      "Step: 42880 loss: 0.148723\n",
      "Step: 42890 loss: 0.189431\n",
      "Step: 42900 loss: 0.168202\n",
      "Step: 42910 loss: 0.214099\n",
      "Step: 42920 loss: 0.111661\n",
      "Step: 42930 loss: 0.104747\n",
      "Step: 42940 loss: 0.11458\n",
      "Step: 42950 loss: 0.194821\n",
      "Step: 42960 loss: 0.18963\n",
      "Step: 42970 loss: 0.208452\n",
      "Step: 42980 loss: 0.193765\n",
      "Step: 42990 loss: 0.248668\n",
      "Step: 43000 loss: 0.104952\n",
      "Saved Model\n",
      "Step: 43010 loss: 0.24477\n",
      "Step: 43020 loss: 0.210817\n",
      "Step: 43030 loss: 0.107364\n",
      "Step: 43040 loss: 0.109544\n",
      "Step: 43050 loss: 0.148576\n",
      "Step: 43060 loss: 0.122235\n",
      "Step: 43070 loss: 0.112254\n",
      "Step: 43080 loss: 0.104693\n",
      "Step: 43090 loss: 0.105432\n",
      "Step: 43100 loss: 0.243311\n",
      "Step: 43110 loss: 0.137926\n",
      "Step: 43120 loss: 0.130241\n",
      "Step: 43130 loss: 0.667293\n",
      "Step: 43140 loss: 0.368926\n",
      "Step: 43150 loss: 0.114756\n",
      "Step: 43160 loss: 0.503561\n",
      "Step: 43170 loss: 0.281482\n",
      "Step: 43180 loss: 0.841969\n",
      "Step: 43190 loss: 0.116483\n",
      "Step: 43200 loss: 0.407683\n",
      "Step: 43210 loss: 0.42083\n",
      "Step: 43220 loss: 0.228363\n",
      "Step: 43230 loss: 0.549043\n",
      "Step: 43240 loss: 0.10623\n",
      "Step: 43250 loss: 0.102861\n",
      "Step: 43260 loss: 0.102129\n",
      "Step: 43270 loss: 0.103517\n",
      "Step: 43280 loss: 0.124503\n",
      "Step: 43290 loss: 0.103955\n",
      "Step: 43300 loss: 0.382454\n",
      "Step: 43310 loss: 0.113129\n",
      "Step: 43320 loss: 0.160498\n",
      "Step: 43330 loss: 0.106109\n",
      "Step: 43340 loss: 0.104731\n",
      "Step: 43350 loss: 0.160595\n",
      "Step: 43360 loss: 0.116636\n",
      "Step: 43370 loss: 0.116371\n",
      "Step: 43380 loss: 0.220668\n",
      "Step: 43390 loss: 0.27462\n",
      "Step: 43400 loss: 0.136433\n",
      "Step: 43410 loss: 0.100416\n",
      "Step: 43420 loss: 0.203304\n",
      "Step: 43430 loss: 0.130918\n",
      "Step: 43440 loss: 0.312527\n",
      "Step: 43450 loss: 0.119825\n",
      "Step: 43460 loss: 0.113862\n",
      "Step: 43470 loss: 0.107163\n",
      "Step: 43480 loss: 0.371987\n",
      "Step: 43490 loss: 0.209422\n",
      "Step: 43500 loss: 0.121528\n",
      "Step: 43510 loss: 0.16868\n",
      "Step: 43520 loss: 0.104954\n",
      "Step: 43530 loss: 0.118009\n",
      "Step: 43540 loss: 0.151057\n",
      "Step: 43550 loss: 0.319176\n",
      "Step: 43560 loss: 0.256278\n",
      "Step: 43570 loss: 0.10247\n",
      "Step: 43580 loss: 0.104795\n",
      "Step: 43590 loss: 0.173965\n",
      "Step: 43600 loss: 0.170698\n",
      "Step: 43610 loss: 0.431614\n",
      "Step: 43620 loss: 0.415695\n",
      "Step: 43630 loss: 0.243121\n",
      "Step: 43640 loss: 0.15173\n",
      "Step: 43650 loss: 0.103322\n",
      "Step: 43660 loss: 0.198665\n",
      "Step: 43670 loss: 0.861183\n",
      "Step: 43680 loss: 0.108689\n",
      "Step: 43690 loss: 0.160548\n",
      "Step: 43700 loss: 0.152168\n",
      "Step: 43710 loss: 0.117962\n",
      "Step: 43720 loss: 0.135108\n",
      "Step: 43730 loss: 0.13778\n",
      "Step: 43740 loss: 0.334571\n",
      "Step: 43750 loss: 0.129488\n",
      "Step: 43760 loss: 0.399401\n",
      "Step: 43770 loss: 0.382744\n",
      "Step: 43780 loss: 0.119925\n",
      "Step: 43790 loss: 0.555174\n",
      "Step: 43800 loss: 0.141397\n",
      "Step: 43810 loss: 0.15954\n",
      "Step: 43820 loss: 0.102296\n",
      "Step: 43830 loss: 0.159176\n",
      "Step: 43840 loss: 0.12752\n",
      "Step: 43850 loss: 0.107762\n",
      "Step: 43860 loss: 0.130078\n",
      "Step: 43870 loss: 0.122513\n",
      "Step: 43880 loss: 0.237121\n",
      "Step: 43890 loss: 0.169442\n",
      "Step: 43900 loss: 0.113822\n",
      "Step: 43910 loss: 0.110652\n",
      "Step: 43920 loss: 0.117476\n",
      "Step: 43930 loss: 0.44267\n",
      "Step: 43940 loss: 0.563057\n",
      "Step: 43950 loss: 0.852403\n",
      "Step: 43960 loss: 0.132675\n",
      "Step: 43970 loss: 0.452335\n",
      "Step: 43980 loss: 0.361338\n",
      "Step: 43990 loss: 0.180015\n",
      "Step: 44000 loss: 0.272953\n",
      "Saved Model\n",
      "Step: 44010 loss: 0.370231\n",
      "Step: 44020 loss: 0.211967\n",
      "Step: 44030 loss: 0.115634\n",
      "Step: 44040 loss: 0.106859\n",
      "Step: 44050 loss: 0.167755\n",
      "Step: 44060 loss: 0.119157\n",
      "Step: 44070 loss: 0.108291\n",
      "Step: 44080 loss: 0.276894\n",
      "Step: 44090 loss: 0.10787\n",
      "Step: 44100 loss: 0.159551\n",
      "Step: 44110 loss: 0.105386\n",
      "Step: 44120 loss: 0.578426\n",
      "Step: 44130 loss: 0.369892\n",
      "Step: 44140 loss: 0.103882\n",
      "Step: 44150 loss: 0.108487\n",
      "Step: 44160 loss: 0.370073\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step: 44170 loss: 0.2035\n",
      "Step: 44180 loss: 0.15654\n",
      "Step: 44190 loss: 0.104065\n",
      "Step: 44200 loss: 0.111314\n",
      "Step: 44210 loss: 0.101138\n",
      "Step: 44220 loss: 0.106676\n",
      "Step: 44230 loss: 0.268077\n",
      "Step: 44240 loss: 0.135465\n",
      "Step: 44250 loss: 0.109786\n",
      "Step: 44260 loss: 0.20021\n",
      "Step: 44270 loss: 0.104894\n",
      "Step: 44280 loss: 0.287074\n",
      "Step: 44290 loss: 0.141661\n",
      "Step: 44300 loss: 0.129823\n",
      "Step: 44310 loss: 0.108479\n",
      "Step: 44320 loss: 0.155135\n",
      "Step: 44330 loss: 0.136926\n",
      "Step: 44340 loss: 0.109277\n",
      "Step: 44350 loss: 0.131163\n",
      "Step: 44360 loss: 0.105929\n",
      "Step: 44370 loss: 0.110991\n",
      "Step: 44380 loss: 0.206221\n",
      "Step: 44390 loss: 0.303211\n",
      "Step: 44400 loss: 0.116069\n",
      "Step: 44410 loss: 0.363999\n",
      "Step: 44420 loss: 0.256382\n",
      "Step: 44430 loss: 0.110059\n",
      "Step: 44440 loss: 0.110923\n",
      "Step: 44450 loss: 0.108623\n",
      "Step: 44460 loss: 0.102245\n",
      "Step: 44470 loss: 0.81267\n",
      "Step: 44480 loss: 0.363868\n",
      "Step: 44490 loss: 0.101172\n",
      "Step: 44500 loss: 0.158603\n",
      "Step: 44510 loss: 0.115151\n",
      "Step: 44520 loss: 0.144479\n",
      "Step: 44530 loss: 0.218534\n",
      "Step: 44540 loss: 0.259262\n",
      "Step: 44550 loss: 0.16159\n",
      "Step: 44560 loss: 0.112308\n",
      "Step: 44570 loss: 0.131077\n",
      "Step: 44580 loss: 0.347694\n",
      "Step: 44590 loss: 0.121432\n",
      "Step: 44600 loss: 0.200741\n",
      "Step: 44610 loss: 0.145679\n",
      "Step: 44620 loss: 0.102889\n",
      "Step: 44630 loss: 0.190684\n",
      "Step: 44640 loss: 0.124952\n",
      "Step: 44650 loss: 0.110264\n",
      "Step: 44660 loss: 0.188938\n",
      "Step: 44670 loss: 0.42026\n",
      "Step: 44680 loss: 0.147481\n",
      "Step: 44690 loss: 0.121299\n",
      "Step: 44700 loss: 0.105218\n",
      "Step: 44710 loss: 0.124216\n",
      "Step: 44720 loss: 0.246708\n",
      "Step: 44730 loss: 0.166406\n",
      "Step: 44740 loss: 0.293099\n",
      "Step: 44750 loss: 0.114595\n",
      "Step: 44760 loss: 0.121995\n",
      "Step: 44770 loss: 0.123872\n",
      "Step: 44780 loss: 0.120131\n",
      "Step: 44790 loss: 0.215504\n",
      "Step: 44800 loss: 0.11771\n",
      "Step: 44810 loss: 0.100811\n",
      "Step: 44820 loss: 0.222467\n",
      "Step: 44830 loss: 0.132356\n",
      "Step: 44840 loss: 0.37945\n",
      "Step: 44850 loss: 0.208815\n",
      "Step: 44860 loss: 0.153026\n",
      "Step: 44870 loss: 0.1523\n",
      "Step: 44880 loss: 0.115824\n",
      "Step: 44890 loss: 0.158447\n",
      "Step: 44900 loss: 0.108861\n",
      "Step: 44910 loss: 0.278916\n",
      "Step: 44920 loss: 0.281\n",
      "Step: 44930 loss: 0.121982\n",
      "Step: 44940 loss: 0.118369\n",
      "Step: 44950 loss: 0.106169\n",
      "Step: 44960 loss: 0.333376\n",
      "Step: 44970 loss: 0.102654\n",
      "Step: 44980 loss: 0.326309\n",
      "Step: 44990 loss: 0.111226\n",
      "Step: 45000 loss: 0.103276\n",
      "Saved Model\n",
      "Step: 45010 loss: 0.104605\n",
      "Step: 45020 loss: 0.107815\n",
      "Step: 45030 loss: 0.132645\n",
      "Step: 45040 loss: 0.290594\n",
      "Step: 45050 loss: 0.110865\n",
      "Step: 45060 loss: 0.107968\n",
      "Step: 45070 loss: 0.105444\n",
      "Step: 45080 loss: 0.136813\n",
      "Step: 45090 loss: 0.105558\n",
      "Step: 45100 loss: 0.104067\n",
      "Step: 45110 loss: 0.138316\n",
      "Step: 45120 loss: 0.578294\n",
      "Step: 45130 loss: 0.110389\n",
      "Step: 45140 loss: 0.193753\n",
      "Step: 45150 loss: 0.215117\n",
      "Step: 45160 loss: 0.101928\n",
      "Step: 45170 loss: 0.0999162\n",
      "Step: 45180 loss: 0.179094\n",
      "Step: 45190 loss: 0.123152\n",
      "Step: 45200 loss: 0.157366\n",
      "Step: 45210 loss: 0.101358\n",
      "Step: 45220 loss: 0.257879\n",
      "Step: 45230 loss: 0.351435\n",
      "Step: 45240 loss: 0.101541\n",
      "Step: 45250 loss: 0.111024\n",
      "Step: 45260 loss: 0.136795\n",
      "Step: 45270 loss: 0.413071\n",
      "Step: 45280 loss: 0.366146\n",
      "Step: 45290 loss: 0.147547\n",
      "Step: 45300 loss: 0.115319\n",
      "Step: 45310 loss: 1.17812\n",
      "Step: 45320 loss: 0.100154\n",
      "Step: 45330 loss: 0.114066\n",
      "Step: 45340 loss: 0.314141\n",
      "Step: 45350 loss: 0.540024\n",
      "Step: 45360 loss: 0.407938\n",
      "Step: 45370 loss: 0.129444\n",
      "Step: 45380 loss: 0.347066\n",
      "Step: 45390 loss: 0.19538\n",
      "Step: 45400 loss: 0.153171\n",
      "Step: 45410 loss: 0.130832\n",
      "Step: 45420 loss: 0.121735\n",
      "Step: 45430 loss: 0.114464\n",
      "Step: 45440 loss: 0.279782\n",
      "Step: 45450 loss: 0.14972\n",
      "Step: 45460 loss: 0.151406\n",
      "Step: 45470 loss: 0.109758\n",
      "Step: 45480 loss: 0.595503\n",
      "Step: 45490 loss: 0.191915\n",
      "Step: 45500 loss: 0.101879\n",
      "Step: 45510 loss: 0.157861\n",
      "Step: 45520 loss: 0.116228\n",
      "Step: 45530 loss: 0.225549\n",
      "Step: 45540 loss: 0.447711\n",
      "Step: 45550 loss: 0.156049\n",
      "Step: 45560 loss: 0.126545\n",
      "Step: 45570 loss: 0.10829\n",
      "Step: 45580 loss: 0.252462\n",
      "Step: 45590 loss: 0.100346\n",
      "Step: 45600 loss: 0.12659\n",
      "Step: 45610 loss: 0.155023\n",
      "Step: 45620 loss: 0.11605\n",
      "Step: 45630 loss: 0.149204\n",
      "Step: 45640 loss: 0.401953\n",
      "Step: 45650 loss: 0.11285\n",
      "Step: 45660 loss: 0.128743\n",
      "Step: 45670 loss: 0.140606\n",
      "Step: 45680 loss: 0.104197\n",
      "Step: 45690 loss: 0.541593\n",
      "Step: 45700 loss: 0.486634\n",
      "Step: 45710 loss: 0.124264\n",
      "Step: 45720 loss: 0.392896\n",
      "Step: 45730 loss: 0.112587\n",
      "Step: 45740 loss: 0.100641\n",
      "Step: 45750 loss: 0.110494\n",
      "Step: 45760 loss: 0.101583\n",
      "Step: 45770 loss: 0.219803\n",
      "Step: 45780 loss: 0.310656\n",
      "Step: 45790 loss: 0.111273\n",
      "Step: 45800 loss: 0.112399\n",
      "Step: 45810 loss: 0.0992327\n",
      "Step: 45820 loss: 0.102287\n",
      "Step: 45830 loss: 0.123411\n",
      "Step: 45840 loss: 0.290198\n",
      "Step: 45850 loss: 0.126762\n",
      "Step: 45860 loss: 0.141138\n",
      "Step: 45870 loss: 0.486111\n",
      "Step: 45880 loss: 0.119721\n",
      "Step: 45890 loss: 0.111041\n",
      "Step: 45900 loss: 0.109641\n",
      "Step: 45910 loss: 0.117781\n",
      "Step: 45920 loss: 0.103227\n",
      "Step: 45930 loss: 0.109952\n",
      "Step: 45940 loss: 0.429738\n",
      "Step: 45950 loss: 0.275026\n",
      "Step: 45960 loss: 0.100794\n",
      "Step: 45970 loss: 0.128486\n",
      "Step: 45980 loss: 0.402106\n",
      "Step: 45990 loss: 0.414285\n",
      "Step: 46000 loss: 0.485796\n",
      "Saved Model\n",
      "Step: 46010 loss: 0.107278\n",
      "Step: 46020 loss: 0.557636\n",
      "Step: 46030 loss: 0.127115\n",
      "Step: 46040 loss: 0.350614\n",
      "Step: 46050 loss: 0.104103\n",
      "Step: 46060 loss: 0.130708\n",
      "Step: 46070 loss: 0.161979\n",
      "Step: 46080 loss: 0.100601\n",
      "Step: 46090 loss: 0.101257\n",
      "Step: 46100 loss: 0.132904\n",
      "Step: 46110 loss: 0.118885\n",
      "Step: 46120 loss: 0.116538\n",
      "Step: 46130 loss: 0.110476\n",
      "Step: 46140 loss: 0.186722\n",
      "Step: 46150 loss: 0.103051\n",
      "Step: 46160 loss: 0.122541\n",
      "Step: 46170 loss: 0.0999147\n",
      "Step: 46180 loss: 0.109727\n",
      "Step: 46190 loss: 0.256611\n",
      "Step: 46200 loss: 0.106978\n",
      "Step: 46210 loss: 0.103312\n",
      "Step: 46220 loss: 0.109067\n",
      "Step: 46230 loss: 0.126681\n",
      "Step: 46240 loss: 0.127034\n",
      "Step: 46250 loss: 0.10581\n",
      "Step: 46260 loss: 0.101122\n",
      "Step: 46270 loss: 0.354593\n",
      "Step: 46280 loss: 0.101809\n",
      "Step: 46290 loss: 0.25082\n",
      "Step: 46300 loss: 0.0993229\n",
      "Step: 46310 loss: 0.620288\n",
      "Step: 46320 loss: 0.104282\n",
      "Step: 46330 loss: 0.132934\n",
      "Step: 46340 loss: 0.241698\n",
      "Step: 46350 loss: 0.190398\n",
      "Step: 46360 loss: 0.13843\n",
      "Step: 46370 loss: 0.165954\n",
      "Step: 46380 loss: 0.442828\n",
      "Step: 46390 loss: 0.170319\n",
      "Step: 46400 loss: 0.121336\n",
      "Step: 46410 loss: 0.225562\n",
      "Step: 46420 loss: 0.174504\n",
      "Step: 46430 loss: 0.187785\n",
      "Step: 46440 loss: 0.654717\n",
      "Step: 46450 loss: 0.10123\n",
      "Step: 46460 loss: 0.102021\n",
      "Step: 46470 loss: 0.241498\n",
      "Step: 46480 loss: 0.0997279\n",
      "Step: 46490 loss: 0.111482\n",
      "Step: 46500 loss: 0.315282\n",
      "Step: 46510 loss: 0.128531\n",
      "Step: 46520 loss: 0.15891\n",
      "Step: 46530 loss: 0.119166\n",
      "Step: 46540 loss: 0.144733\n",
      "Step: 46550 loss: 0.104574\n",
      "Step: 46560 loss: 0.275634\n",
      "Step: 46570 loss: 0.1068\n",
      "Step: 46580 loss: 0.141532\n",
      "Step: 46590 loss: 0.102549\n",
      "Step: 46600 loss: 0.100129\n",
      "Step: 46610 loss: 0.112064\n",
      "Step: 46620 loss: 0.0988379\n",
      "Step: 46630 loss: 0.177458\n",
      "Step: 46640 loss: 0.160844\n",
      "Step: 46650 loss: 0.102143\n",
      "Step: 46660 loss: 0.13966\n",
      "Step: 46670 loss: 0.101369\n",
      "Step: 46680 loss: 0.208467\n",
      "Step: 46690 loss: 0.147068\n",
      "Step: 46700 loss: 0.10049\n",
      "Step: 46710 loss: 0.100422\n",
      "Step: 46720 loss: 0.185229\n",
      "Step: 46730 loss: 0.114979\n",
      "Step: 46740 loss: 0.13408\n",
      "Step: 46750 loss: 0.597552\n",
      "Step: 46760 loss: 0.151057\n",
      "Step: 46770 loss: 0.103238\n",
      "Step: 46780 loss: 0.10206\n",
      "Step: 46790 loss: 0.354068\n",
      "Step: 46800 loss: 0.207715\n",
      "Step: 46810 loss: 0.291454\n",
      "Step: 46820 loss: 0.136042\n",
      "Step: 46830 loss: 0.109156\n",
      "Step: 46840 loss: 0.148489\n",
      "Step: 46850 loss: 0.110817\n",
      "Step: 46860 loss: 0.105348\n",
      "Step: 46870 loss: 0.367898\n",
      "Step: 46880 loss: 0.196501\n",
      "Step: 46890 loss: 0.114032\n",
      "Step: 46900 loss: 0.106164\n",
      "Step: 46910 loss: 0.127246\n",
      "Step: 46920 loss: 0.190679\n",
      "Step: 46930 loss: 0.131751\n",
      "Step: 46940 loss: 0.338498\n",
      "Step: 46950 loss: 0.481241\n",
      "Step: 46960 loss: 0.22869\n",
      "Step: 46970 loss: 0.146128\n",
      "Step: 46980 loss: 0.105942\n",
      "Step: 46990 loss: 0.233962\n",
      "Step: 47000 loss: 0.210251\n",
      "Saved Model\n",
      "Step: 47010 loss: 0.263085\n",
      "Step: 47020 loss: 0.116006\n",
      "Step: 47030 loss: 0.174478\n",
      "Step: 47040 loss: 0.180666\n",
      "Step: 47050 loss: 0.0998101\n",
      "Step: 47060 loss: 0.470483\n",
      "Step: 47070 loss: 0.100678\n",
      "Step: 47080 loss: 0.124121\n",
      "Step: 47090 loss: 0.113894\n",
      "Step: 47100 loss: 0.0992647\n",
      "Step: 47110 loss: 0.282218\n",
      "Step: 47120 loss: 0.185665\n",
      "Step: 47130 loss: 0.289076\n",
      "Step: 47140 loss: 0.746313\n",
      "Step: 47150 loss: 0.322146\n",
      "Step: 47160 loss: 0.289319\n",
      "Step: 47170 loss: 0.26815\n",
      "Step: 47180 loss: 0.11285\n",
      "Step: 47190 loss: 0.197229\n",
      "Step: 47200 loss: 0.327767\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step: 47210 loss: 0.112841\n",
      "Step: 47220 loss: 0.641932\n",
      "Step: 47230 loss: 0.261087\n",
      "Step: 47240 loss: 0.240021\n",
      "Step: 47250 loss: 0.211281\n",
      "Step: 47260 loss: 0.100937\n",
      "Step: 47270 loss: 0.108555\n",
      "Step: 47280 loss: 0.105594\n",
      "Step: 47290 loss: 0.103131\n",
      "Step: 47300 loss: 0.139107\n",
      "Step: 47310 loss: 0.109371\n",
      "Step: 47320 loss: 0.418391\n",
      "Step: 47330 loss: 0.0988733\n",
      "Step: 47340 loss: 0.167312\n",
      "Step: 47350 loss: 0.17647\n",
      "Step: 47360 loss: 0.14746\n",
      "Step: 47370 loss: 0.219458\n",
      "Step: 47380 loss: 0.475655\n",
      "Step: 47390 loss: 0.111831\n",
      "Step: 47400 loss: 0.113887\n",
      "Step: 47410 loss: 0.0987209\n",
      "Step: 47420 loss: 0.184824\n",
      "Step: 47430 loss: 0.175491\n",
      "Step: 47440 loss: 0.797833\n",
      "Step: 47450 loss: 0.103924\n",
      "Step: 47460 loss: 0.184284\n",
      "Step: 47470 loss: 0.179754\n",
      "Step: 47480 loss: 0.230305\n",
      "Step: 47490 loss: 0.115631\n",
      "Step: 47500 loss: 0.097848\n",
      "Step: 47510 loss: 0.305652\n",
      "Step: 47520 loss: 0.122897\n",
      "Step: 47530 loss: 0.117728\n",
      "Step: 47540 loss: 0.112769\n",
      "Step: 47550 loss: 0.310324\n",
      "Step: 47560 loss: 0.0983254\n",
      "Step: 47570 loss: 0.122028\n",
      "Step: 47580 loss: 0.10171\n",
      "Step: 47590 loss: 0.389124\n",
      "Step: 47600 loss: 0.344834\n",
      "Step: 47610 loss: 0.129531\n",
      "Step: 47620 loss: 0.188473\n",
      "Step: 47630 loss: 0.144498\n",
      "Step: 47640 loss: 0.148145\n",
      "Step: 47650 loss: 0.157924\n",
      "Step: 47660 loss: 0.157198\n",
      "Step: 47670 loss: 0.130609\n",
      "Step: 47680 loss: 0.126715\n",
      "Step: 47690 loss: 0.123822\n",
      "Step: 47700 loss: 0.138578\n",
      "Step: 47710 loss: 0.36526\n",
      "Step: 47720 loss: 0.133271\n",
      "Step: 47730 loss: 0.102602\n",
      "Step: 47740 loss: 0.13746\n",
      "Step: 47750 loss: 0.147564\n",
      "Step: 47760 loss: 0.13227\n",
      "Step: 47770 loss: 0.108358\n",
      "Step: 47780 loss: 0.224933\n",
      "Step: 47790 loss: 0.1702\n",
      "Step: 47800 loss: 0.18684\n",
      "Step: 47810 loss: 0.102196\n",
      "Step: 47820 loss: 0.130429\n",
      "Step: 47830 loss: 0.11625\n",
      "Step: 47840 loss: 0.116883\n",
      "Step: 47850 loss: 0.126807\n",
      "Step: 47860 loss: 0.119757\n",
      "Step: 47870 loss: 0.13718\n",
      "Step: 47880 loss: 0.108008\n",
      "Step: 47890 loss: 0.179973\n",
      "Step: 47900 loss: 0.102374\n",
      "Step: 47910 loss: 0.253849\n",
      "Step: 47920 loss: 0.0993639\n",
      "Step: 47930 loss: 0.351678\n",
      "Step: 47940 loss: 0.259601\n",
      "Step: 47950 loss: 0.338233\n",
      "Step: 47960 loss: 0.0999845\n",
      "Step: 47970 loss: 0.635316\n",
      "Step: 47980 loss: 0.145825\n",
      "Step: 47990 loss: 0.363667\n",
      "Step: 48000 loss: 0.154107\n",
      "Saved Model\n"
     ]
    }
   ],
   "source": [
    "num_epochs = 20\n",
    "for i in range(num_epochs):\n",
    "    for j in range(num_batches):\n",
    "        path_dict = [path1_len[j*batch_size:(j+1)*batch_size], path2_len[j*batch_size:(j+1)*batch_size]]\n",
    "        word_dict = [word_p1_ids[j*batch_size:(j+1)*batch_size], word_p2_ids[j*batch_size:(j+1)*batch_size]]\n",
    "        pos_dict = [pos_p1_ids[j*batch_size:(j+1)*batch_size], pos_p2_ids[j*batch_size:(j+1)*batch_size]]\n",
    "        dep_dict = [dep_p1_ids[j*batch_size:(j+1)*batch_size], dep_p2_ids[j*batch_size:(j+1)*batch_size]]\n",
    "        y_dict = rel_ids[j*batch_size:(j+1)*batch_size]\n",
    "        \n",
    "        feed_dict = {\n",
    "            path_length:path_dict,\n",
    "            word_ids:word_dict,\n",
    "            pos_ids:pos_dict,\n",
    "            dep_ids:dep_dict,\n",
    "            y:y_dict}\n",
    "        _, loss, step = sess.run([optimizer, total_loss, global_step], feed_dict)\n",
    "        if step%10==0:\n",
    "            print(\"Step:\", step, \"loss:\",loss)\n",
    "        if step % 1000 == 0:\n",
    "            saver.save(sess, model_dir + '/model')\n",
    "            print(\"Saved Model\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "training accuracy 96.075\n"
     ]
    }
   ],
   "source": [
    "# training accuracy\n",
    "all_predictions = []\n",
    "for j in range(num_batches):\n",
    "    path_dict = [path1_len[j*batch_size:(j+1)*batch_size], path2_len[j*batch_size:(j+1)*batch_size]]\n",
    "    word_dict = [word_p1_ids[j*batch_size:(j+1)*batch_size], word_p2_ids[j*batch_size:(j+1)*batch_size]]\n",
    "    pos_dict = [pos_p1_ids[j*batch_size:(j+1)*batch_size], pos_p2_ids[j*batch_size:(j+1)*batch_size]]\n",
    "    dep_dict = [dep_p1_ids[j*batch_size:(j+1)*batch_size], dep_p2_ids[j*batch_size:(j+1)*batch_size]]\n",
    "    y_dict = rel_ids[j*batch_size:(j+1)*batch_size]\n",
    "\n",
    "    feed_dict = {\n",
    "        path_length:path_dict,\n",
    "        word_ids:word_dict,\n",
    "        pos_ids:pos_dict,\n",
    "        dep_ids:dep_dict,\n",
    "        y:y_dict}\n",
    "    batch_predictions = sess.run(predictions, feed_dict)\n",
    "    all_predictions.append(batch_predictions)\n",
    "\n",
    "y_pred = []\n",
    "for i in range(num_batches):\n",
    "    for pred in all_predictions[i]:\n",
    "        y_pred.append(pred)\n",
    "\n",
    "count = 0\n",
    "for i in range(batch_size*num_batches):\n",
    "    count += y_pred[i]==rel_ids[i]\n",
    "accuracy = count/(batch_size*num_batches) * 100\n",
    "\n",
    "print(\"training accuracy\", accuracy)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "f = open(data_dir + '/test_paths', 'rb')\n",
    "word_p1, word_p2, dep_p1, dep_p2, pos_p1, pos_p2 = pickle.load(f)\n",
    "f.close()\n",
    "\n",
    "relations = []\n",
    "for line in open(data_dir + '/test_relations.txt'):\n",
    "    relations.append(line.strip().split()[0])\n",
    "\n",
    "length = len(word_p1)\n",
    "num_batches = int(length/batch_size)\n",
    "\n",
    "for i in range(length):\n",
    "    for j, word in enumerate(word_p1[i]):\n",
    "        word = word.lower()\n",
    "        word_p1[i][j] = word if word in word2id else unknown_token \n",
    "    for k, word in enumerate(word_p2[i]):\n",
    "        word = word.lower()\n",
    "        word_p2[i][k] = word if word in word2id else unknown_token \n",
    "    for l, d in enumerate(dep_p1[i]):\n",
    "        dep_p1[i][l] = d if d in dep2id else 'OTH'\n",
    "    for m, d in enumerate(dep_p2[i]):\n",
    "        dep_p2[i][m] = d if d in dep2id else 'OTH'\n",
    "\n",
    "word_p1_ids = np.ones([length, max_len_path],dtype=int)\n",
    "word_p2_ids = np.ones([length, max_len_path],dtype=int)\n",
    "pos_p1_ids = np.ones([length, max_len_path],dtype=int)\n",
    "pos_p2_ids = np.ones([length, max_len_path],dtype=int)\n",
    "dep_p1_ids = np.ones([length, max_len_path],dtype=int)\n",
    "dep_p2_ids = np.ones([length, max_len_path],dtype=int)\n",
    "rel_ids = np.array([rel2id[rel] for rel in relations])\n",
    "path1_len = np.array([len(w) for w in word_p1], dtype=int)\n",
    "path2_len = np.array([len(w) for w in word_p2])\n",
    "\n",
    "for i in range(length):\n",
    "    for j, w in enumerate(word_p1[i]):\n",
    "        word_p1_ids[i][j] = word2id[w]\n",
    "    for j, w in enumerate(word_p2[i]):\n",
    "        word_p2_ids[i][j] = word2id[w]\n",
    "    for j, w in enumerate(pos_p1[i]):\n",
    "        pos_p1_ids[i][j] = pos_tag(w)\n",
    "    for j, w in enumerate(pos_p2[i]):\n",
    "        pos_p2_ids[i][j] = pos_tag(w)\n",
    "    for j, w in enumerate(dep_p1[i]):\n",
    "        dep_p1_ids[i][j] = dep2id[w]\n",
    "    for j, w in enumerate(dep_p2[i]):\n",
    "        dep_p2_ids[i][j] = dep2id[w]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "test accuracy 63.2472324723\n"
     ]
    }
   ],
   "source": [
    "# test \n",
    "all_predictions = []\n",
    "for j in range(num_batches):\n",
    "    path_dict = [path1_len[j*batch_size:(j+1)*batch_size], path2_len[j*batch_size:(j+1)*batch_size]]\n",
    "    word_dict = [word_p1_ids[j*batch_size:(j+1)*batch_size], word_p2_ids[j*batch_size:(j+1)*batch_size]]\n",
    "    pos_dict = [pos_p1_ids[j*batch_size:(j+1)*batch_size], pos_p2_ids[j*batch_size:(j+1)*batch_size]]\n",
    "    dep_dict = [dep_p1_ids[j*batch_size:(j+1)*batch_size], dep_p2_ids[j*batch_size:(j+1)*batch_size]]\n",
    "    y_dict = rel_ids[j*batch_size:(j+1)*batch_size]\n",
    "\n",
    "    feed_dict = {\n",
    "        path_length:path_dict,\n",
    "        word_ids:word_dict,\n",
    "        pos_ids:pos_dict,\n",
    "        dep_ids:dep_dict,\n",
    "        y:y_dict}\n",
    "    batch_predictions = sess.run(predictions, feed_dict)\n",
    "    all_predictions.append(batch_predictions)\n",
    "\n",
    "y_pred = []\n",
    "for i in range(num_batches):\n",
    "    for pred in all_predictions[i]:\n",
    "        y_pred.append(pred)\n",
    "\n",
    "count = 0\n",
    "for i in range(batch_size*num_batches):\n",
    "    count += y_pred[i]==rel_ids[i]\n",
    "accuracy = count/(batch_size*num_batches) * 100\n",
    "\n",
    "print(\"test accuracy\", accuracy)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
